From 1d924073c4e753f1ddb1ec938317c84f4183b6a6 Mon Sep 17 00:00:00 2001 From: Peter Bhat Harkins Date: Sat, 12 Aug 2023 08:44:21 -0500 Subject: [PATCH] robots.txt: disallow GPT scraper --- public/robots.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/public/robots.txt b/public/robots.txt index 2809603e..6a97d299 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -3,3 +3,6 @@ Crawl-delay: 1 Disallow: /search Disallow: /page/ Disallow: /comments/page/ + +User-agent: GPTBot +Disallow: /