Avoid overwriting an existing robots.txt

lib/jekyll/jekyll-sitemap.rb

@@ -62,11 +62,11 @@ def robots
 
     # Checks if a file already exists in the site source
     def file_exists?(file_path)
-      if @site.respond_to?(:in_source_dir)
-        File.exist? @site.in_source_dir(file_path)
-      else
-        File.exist? Jekyll.sanitized_path(@site.source, file_path)
-      end
+      pages_and_files.any? { |p| p.url == "/#{file_path}" }
+    end
+
+    def pages_and_files
+      @pages_and_files ||= @site.pages + @site.static_files
     end
   end
 end

spec/jekyll-sitemap_spec.rb

@@ -195,7 +195,7 @@
         expect(contents).not_to match(%r!\ATHIS IS MY LAYOUT!)
       end
 
-      it "creates a sitemap.xml file" do
+      it "creates a robots.txt file" do
         expect(File.exist?(dest_dir("robots.txt"))).to be_truthy
       end
 
@@ -204,4 +204,53 @@
       end
     end
   end
+
+  context "with user-defined robots.txt" do
+    let(:fixture) { "/" }
+    let(:fixture_source) { robot_fixtures(fixture) }
+    let(:fixture_dest) { robot_fixtures(fixture, "_site") }
+    let(:robot_contents) { File.read(robot_fixtures(fixture, "_site", "robots.txt")).strip }
+    let(:overrides) do
+      {
+        "source"      => fixture_source,
+        "destination" => fixture_dest,
+        "url"         => "http://example.org",
+      }
+    end
+
+    before(:each) { setup_fixture(fixture) }
+    after(:each) { cleanup_fixture(fixture) }
+
+    context "as a static-file at source-root" do
+      let(:fixture) { "static-at-source-root" }
+
+      it "doesn't override the robots file" do
+        expect(robot_contents).to eql("Allow: /")
+      end
+    end
+
+    context "as a static-file in a subdir" do
+      let(:fixture) { "static-in-subdir" }
+
+      it "generates a valid robot.txt" do
+        expect(robot_contents).to eql("Sitemap: http://example.org/sitemap.xml")
+      end
+    end
+
+    context "as a page at root" do
+      let(:fixture) { "page-at-root" }
+
+      it "doesn't override the robots file" do
+        expect(robot_contents).to eql("Allow: http://example.org")
+      end
+    end
+
+    context "as a page with permalink in a subdir" do
+      let(:fixture) { "permalinked-page-in-subdir" }
+
+      it "doesn't override the robots file" do
+        expect(robot_contents).to eql("Allow: http://example.org")
+      end
+    end
+  end
 end

spec/robot-fixtures/page-at-root/robots.txt

@@ -0,0 +1,4 @@
+---
+---
+
+Allow: {{ site.url }}

spec/robot-fixtures/permalinked-page-in-subdir/assets/robots.txt

@@ -0,0 +1,5 @@
+---
+permalink: '/robots.txt'
+---
+
+Allow: {{ site.url }}

spec/robot-fixtures/static-at-source-root/robots.txt

@@ -0,0 +1 @@
+Allow: /

spec/robot-fixtures/static-in-subdir/assets/robots.txt

@@ -0,0 +1 @@
+Allow: /

spec/spec_helper.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "jekyll"
+require "fileutils"
 require File.expand_path("../lib/jekyll-sitemap", __dir__)
 
 Jekyll.logger.log_level = :error
@@ -12,6 +13,8 @@
 
   SOURCE_DIR = File.expand_path("fixtures", __dir__)
   DEST_DIR   = File.expand_path("dest", __dir__)
+  ROBOT_FIXTURES = File.expand_path("robot-fixtures", __dir__)
+  ROBOT_FIXTURE_ITEMS = %w(_posts _layouts _config.yml index.html).freeze
 
   def source_dir(*files)
     File.join(SOURCE_DIR, *files)
@@ -20,4 +23,18 @@ def source_dir(*files)
   def dest_dir(*files)
     File.join(DEST_DIR, *files)
   end
+
+  def robot_fixtures(*subdirs)
+    File.join(ROBOT_FIXTURES, *subdirs)
+  end
+
+  def setup_fixture(directory)
+    ROBOT_FIXTURE_ITEMS.each { |item| FileUtils.cp_r(source_dir(item), robot_fixtures(directory)) }
+  end
+
+  def cleanup_fixture(directory, dest_dirname = "_site")
+    (ROBOT_FIXTURE_ITEMS + [dest_dirname]).each do |item|
+      FileUtils.remove_entry(robot_fixtures(directory, item))
+    end
+  end
 end