<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0"
	xmlns:content="http://purl.org/rss/1.0/modules/content/"
	xmlns:wfw="http://wellformedweb.org/CommentAPI/"
	xmlns:dc="http://purl.org/dc/elements/1.1/"
	xmlns:atom="http://www.w3.org/2005/Atom"
	xmlns:sy="http://purl.org/rss/1.0/modules/syndication/"
	xmlns:slash="http://purl.org/rss/1.0/modules/slash/"
	>

<channel>
	<title>Kerberos&#039; Web &#187; Robots.txt</title>
	<atom:link href="http://k3rb.com/blog/kerb_tag/robots-txt/feed/" rel="self" type="application/rss+xml" />
	<link>http://k3rb.com/blog</link>
	<description></description>
	<lastBuildDate>Fri, 30 Jul 2010 14:08:06 +0000</lastBuildDate>
	<generator>http://wordpress.org/?v=2.9.1</generator>
	<language>en</language>
	<sy:updatePeriod>hourly</sy:updatePeriod>
	<sy:updateFrequency>1</sy:updateFrequency>
			<item>
		<title>Preventing Web crawlers from accessing your development site with robots.txt</title>
		<link>http://k3rb.com/blog/2009/10/preventing-web-crawlers-from-accessing-your-development-site-with-robots-txt/</link>
		<comments>http://k3rb.com/blog/2009/10/preventing-web-crawlers-from-accessing-your-development-site-with-robots-txt/#comments</comments>
		<pubDate>Sat, 31 Oct 2009 13:58:14 +0000</pubDate>
		<dc:creator>Kerberos</dc:creator>
				<category><![CDATA[Useful tools]]></category>
		<category><![CDATA[Web development]]></category>
		<category><![CDATA[Robots.txt]]></category>

		<guid isPermaLink="false">http://k3rb.com/blog/?p=68</guid>
		<description><![CDATA[We recently started a development site for a new project.  Soon, we noticed an Amazon AWS instance accessing URLs deep within our site regularly.  The site was access protected (you need to be logged in to see the pages), so those access attempts failed but were annoying nonetheless.
A bit more digging revealed that this was [...]]]></description>
		<wfw:commentRss>http://k3rb.com/blog/2009/10/preventing-web-crawlers-from-accessing-your-development-site-with-robots-txt/feed/</wfw:commentRss>
		<slash:comments>0</slash:comments>
		</item>
	</channel>
</rss>
