If you run WordPress on Windows (and who doesn’t?) and have the need to block specific user agents (bots, crawlers, browsers) below is a decent way I’ve found that uses rewriting rules and works along side the needed WordPress rules:
Adding this rule to your web.config will block the request from the specified agent(s):
<rule name="RequestBlockingRule1" stopProcessing="true">
<match url=".*" />
<conditions>
<add input="{HTTP_USER_AGENT}" pattern="agent1|agent2|agent3" />
</conditions>
<action type="CustomResponse" statusCode="403"
statusReason="Forbidden: Access is denied."
statusDescription="You do not have permission to view this page." />
</rule>
So your entire web.config may look like this:
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<system.webServer>
<rewrite>
<rules>
<rule name="RequestBlockingRule1" stopProcessing="true">
<match url=".*" />
<conditions>
<add input="{HTTP_USER_AGENT}" pattern="feedburner|android" />
</conditions>
<action type="CustomResponse" statusCode="403"
statusReason="Forbidden: Access is denied."
statusDescription="You do not have permission to view this page." />
</rule>
<rule name="WordPress Rule 1" stopProcessing="true">
<match url="^index\.php$" ignoreCase="false" />
<action type="None" />
</rule>
<rule name="WordPress Rule 2" stopProcessing="true">
<match url="^([_0-9a-zA-Z-]+/)?wp-admin$" ignoreCase="false" />
<action type="Redirect" url="{R:1}wp-admin/" redirectType="Permanent" />
</rule>
<rule name="WordPress Rule 3" stopProcessing="true">
<match url="^" ignoreCase="false" />
<conditions logicalGrouping="MatchAny">
<add input="{REQUEST_FILENAME}" matchType="IsFile" ignoreCase="false" />
<add input="{REQUEST_FILENAME}" matchType="IsDirectory" ignoreCase="false" />
</conditions>
<action type="None" />
</rule>
<rule name="WordPress Rule 4" stopProcessing="true">
<match url="^" ignoreCase="false" />
<conditions logicalGrouping="MatchAny">
<add input="{REQUEST_FILENAME}" matchType="IsFile" ignoreCase="false" />
<add input="{REQUEST_FILENAME}" matchType="IsDirectory" ignoreCase="false" />
<add input="{URL}" pattern="([a-zA-Z0-9\./_-]+)\.axd" />
</conditions>
<action type="None" />
</rule>
<rule name="WordPress Rule 5" stopProcessing="true">
<match url="^[_0-9a-zA-Z-]+/(wp-(content|admin|includes).*)" ignoreCase="false" />
<action type="Rewrite" url="{R:1}" />
</rule>
<rule name="WordPress Rule 6" stopProcessing="true">
<match url="." ignoreCase="false" />
<action type="Rewrite" url="index.php" />
</rule>
</rules>
</rewrite>
</system.webServer>
</configuration>