Changes
ignore.txt
lib/CrawlerCommons/ParseState.pm
lib/CrawlerCommons/RobotDirective.pm
lib/CrawlerCommons/RobotRules.pm
lib/CrawlerCommons/RobotRulesParser.pm
lib/CrawlerCommons/RobotToken.pm
Makefile.PL
MANIFEST			This list of files
META.yml
README
README.md
t/00-load.t
t/manifest.t
t/pod-coverage.t
t/pod.t
t/robot_rules_parser.t
t/robots/directive-typos-robots.txt
t/robots/dos-line-endings.txt
t/robots/extended-standard-robots.txt
t/robots/format-errors-robots.txt
t/robots/malformed-path.txt
t/robots/many-user-agents.txt
t/robots/relative-sitemap-robots.txt
t/robots/robots-with-utf16be-bom.txt
t/robots/robots-with-utf16le-bom.txt
t/robots/robots-with-utf8-bom.txt
t/robots/sitemap-robots.txt
t/robots/wildcards.txt
t/robots/www.flot.com-robots.txt
