ACLEDCrawlingScraping/general-scraper-def.xml at master · jackpay/ACLEDCrawlingScraping · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
<jqm>
    <jar>
<!--        <path>jqm-demo/jqm-test-pyl/jqm-test-pyl.jar</path>-->
    <path>/home/jack/springcrawler/target/acled-spring-crawler-1.2.6.jar</path>

        <jobdefinitions>
            <jobDefinition>
                <name>SpringCollector</name>
                <description>Crawler that user the Spring architecture to output crawled docs.</description>
                <canBeRestarted>true</canBeRestarted>
                <javaClassName>com.casm.acled.crawler.spring.SpringCrawler</javaClassName>
                <application>incremental-crawling</application>
                <module>Crawling</module>
                <keyword1></keyword1>
                <keyword2></keyword2>
                <keyword3></keyword3>
                <highlander>false</highlander>
                <parameters>
                    <parameter>
                        <key>casm.jqm.polling.agent</key>
                        <value>casm.jqm.polling.agent http://www.taglaboratory.org/</value>
                    </parameter>
                    <parameter>
                        <key>casm.jqm.polling.filter</key>
                        <value>casm.jqm.polling.filter .*</value>
                    </parameter>
                    <parameter>
                        <key>casm.jqm.polling.crawldb</key>
                        <value>casm.jqm.polling.crawldb /var/www/acled/jqm/crawl-databases</value>
                    </parameter>
                    <parameter>
                        <key>casm.jqm.polling.depth</key>
                        <value>casm.jqm.polling.depth 8</value>
                    </parameter>
                    <parameter>
                        <key>casm.jqm.polling.politeness</key>
                        <value>casm.jqm.polling.politeness 350</value>
                    </parameter>
                    <parameter>
                        <key>casm.jqm.polling.threads</key>
                        <value>casm.jqm.polling.threads 2</value>
                    </parameter>
                    <parameter>
                        <key>casm.jqm.polling.robots</key>
                        <value>casm.jqm.polling.robots false</value>
                    </parameter>
                    <parameter>
                        <key>casm.jqm.polling.sitemap</key>
                        <value>casm.jqm.polling.sitemap false</value>
                    </parameter>
                    <parameter>
                        <key>casm.jqm.crawling.index</key>
                        <value>casm.jqm.crawling.index</value>
                    </parameter>
                    <parameter>
                        <key>casm.jqm.scraping.scrapers</key>
                        <value>casm.jqm.scraping.scrapers /var/www/acled/jqm/example_scrapers</value>
                    </parameter>
                </parameters>
            </jobDefinition>
        </jobdefinitions>
    </jar>
</jqm>