Add sitemap to doc generator

t push -f# with '#' will be ignored, and an empty message aborts the commit.

Makefile

@@ -15,14 +15,15 @@
 
 LLVM_CONFIG ?= ## llvm-config command path to use
 
-release ?=      ## Compile in release mode
-stats ?=        ## Enable statistics output
-progress ?=     ## Enable progress output
-threads ?=      ## Maximum number of threads to use
-debug ?=        ## Add symbolic debug info
-verbose ?=      ## Run specs in verbose mode
-junit_output ?= ## Directory to output junit results
-static ?=       ## Enable static linking
+release ?=        ## Compile in release mode
+stats ?=          ## Enable statistics output
+progress ?=       ## Enable progress output
+threads ?=        ## Maximum number of threads to use
+debug ?=          ## Add symbolic debug info
+verbose ?=        ## Run specs in verbose mode
+junit_output ?=   ## Directory to output junit results
+static ?=         ## Enable static linking
+version := master ## Specify version for docs generator
 
 O := .build
 SOURCES := $(shell find src -name '*.cr')
@@ -64,6 +65,12 @@ DEPS = $(LLVM_EXT_OBJ) $(LIB_CRYSTAL_TARGET)
 CFLAGS += -fPIC $(if $(debug),-g -O0)
 CXXFLAGS += $(if $(debug),-g -O0)
 
+ifeq ($(version),master)
+  DOCS_OPTIONS := --sitemap-base-url=https://crystal-lang.org/api/master --sitemap-changefreq=daily --sitemap-priority=0.3
+else
+  DOCS_OPTIONS := --sitemap-base-url=https://crystal-lang.org/api/$(version) --sitemap-changefreq=never --sitemap-priority=1.0
+endif
+
 ifeq (${LLVM_CONFIG},)
   $(error Could not locate llvm-config, make sure it is installed and in your PATH, or set LLVM_CONFIG)
 else
@@ -104,7 +111,7 @@ compiler_spec: $(O)/compiler_spec ## Run compiler specs
 
 .PHONY: docs
 docs: ## Generate standard library documentation
-	$(BUILD_PATH) ./bin/crystal docs src/docs_main.cr
+	$(BUILD_PATH) ./bin/crystal docs src/docs_main.cr $(DOCS_OPTIONS)
 
 .PHONY: crystal
 crystal: $(O)/crystal ## Build the compiler

man/crystal.1

@@ -175,8 +175,12 @@ Options:
 .Pp
 .It Fl o Ar DIR, Fl -output Ar DIR
 Set the output directory (default: ./docs).
-.It Fl b Ar URL, Fl -canonical-base-url Ar URL
-Set the canonical base URL.
+.It Fl b Ar URL, Fl -sitemap-base-url Ar URL
+Set the sitemap base URL. Sitemap will only be generated when this option is set.
+.It Fl -sitemap-priority Ar PRIO
+Set the priority assigned to sitemap entries (default: 1.0).
+.It Fl -sitemap-changefreq Ar FREQ
+Set the changefreq assigned to sitemap entries (default: never).
 .El
 .Pp
 .It

spec/compiler/crystal/tools/doc/generator_spec.cr

@@ -244,4 +244,22 @@ describe Doc::Generator do
       doc_method.doc.not_nil!.should_not contain %(NOTE: This is a pseudo-method)
     end
   end
+
+  it "generates sitemap" do
+    program = Program.new
+    generator = Doc::Generator.new program, ["."]
+    doc_type = Doc::Type.new generator, program
+
+    Doc::SitemapTemplate.new([doc_type], "http://example.com/api/1.0", "0.8", "monthly").to_s.should eq <<-XML
+      <?xml version="1.0" encoding="UTF-8"?>
+      <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <url>
+          <loc>http://example.com/api/1.0/toplevel.html</loc>
+          <priority>0.8</priority>
+          <changefreq>monthly</changefreq>
+        </url>
+      </urlset>
+
+      XML
+  end
 end

src/compiler/crystal/command/docs.cr

@@ -9,6 +9,9 @@ class Crystal::Command
   private def docs
     output_format = "html"
     output_directory = File.join(".", "docs")
+    sitemap_base_url = nil
+    sitemap_priority = "1.0"
+    sitemap_changefreq = "never"
 
     compiler = Compiler.new
 
@@ -27,12 +30,27 @@ class Crystal::Command
       opts.on("--format=FORMAT", "-f FORMAT", "Set the output format [#{VALID_OUTPUT_FORMATS.join(", ")}] (default: #{output_format})") do |value|
         if !VALID_OUTPUT_FORMATS.includes? value
           STDERR.puts "Invalid format '#{value}'"
-          puts opts
-          exit
+          abort opts
         end
         output_format = value
       end
 
+      opts.on("--canonical-base-url=URL", "Deprecated option. Use --sitemap-base-url instead.") do |value|
+        abort "Option --canonical-base-url is no longer supported.  Use --sitemap-base-url instead."
+      end
+
+      opts.on("--sitemap-base-url=URL", "-b URL", "Set the sitemap base URL and generates sitemap") do |value|
+        sitemap_base_url = value
+      end
+
+      opts.on("--sitemap-priority=PRIO", "Set the sitemap priority (default: 1.0)") do |value|
+        sitemap_priority = value
+      end
+
+      opts.on("--sitemap-changefreq=FREQ", "Set the sitemap changefreq (default: never)") do |value|
+        sitemap_changefreq = value
+      end
+
       opts.on("-D FLAG", "--define FLAG", "Define a compile-time flag") do |flag|
         compiler.flags << flag
       end
@@ -83,6 +101,6 @@ class Crystal::Command
     compiler.wants_doc = true
     result = compiler.top_level_semantic sources
 
-    Doc::Generator.new(result.program, included_dirs, output_directory, output_format).run
+    Doc::Generator.new(result.program, included_dirs, output_directory, output_format, sitemap_base_url, sitemap_priority, sitemap_changefreq).run
   end
 end

src/compiler/crystal/tools/doc/generator.cr

@@ -29,11 +29,13 @@ class Crystal::Doc::Generator
   }
 
   def self.new(program : Program, included_dirs : Array(String))
-    new(program, included_dirs, ".", "html", nil)
+    new(program, included_dirs, ".", "html", nil, "1.0", "never")
   end
 
   def initialize(@program : Program, @included_dirs : Array(String),
-                 @output_dir : String, @output_format : String)
+                 @output_dir : String, @output_format : String,
+                 @sitemap_base_url : String?,
+                 @sitemap_priority : String, @sitemap_changefreq : String)
     @base_dir = Dir.current.chomp
     @types = {} of Crystal::Type => Doc::Type
     @repo_name = ""
@@ -88,6 +90,7 @@ class Crystal::Doc::Generator
     copy_files
     generate_types_docs types, @output_dir, types
     generate_readme program_type, types
+    generate_sitemap types
   end
 
   def generate_readme(program_type, types)
@@ -101,6 +104,12 @@ class Crystal::Doc::Generator
     File.write File.join(@output_dir, "search-index.js"), main_index.to_jsonp
   end
 
+  def generate_sitemap(types)
+    if sitemap_base_url = @sitemap_base_url
+      File.write File.join(@output_dir, "sitemap.xml"), SitemapTemplate.new(types, sitemap_base_url, "1.0", "never")
+    end
+  end
+
   def copy_files
     Dir.mkdir_p File.join(@output_dir, "css")
     Dir.mkdir_p File.join(@output_dir, "js")

src/compiler/crystal/tools/doc/html/sitemap.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <% types.each do |type| %><url>
+    <loc><%= ::Path.posix(base_url, type.path) %></loc>
+    <priority><%= priority %></priority>
+    <changefreq><%= changefreq %></changefreq>
+  </url><% end %>
+</urlset>

src/compiler/crystal/tools/doc/templates.cr

@@ -56,4 +56,8 @@ module Crystal::Doc
   struct StyleTemplate
     ECR.def_to_s "#{__DIR__}/html/css/style.css"
   end
+
+  record SitemapTemplate, types : Array(Type), base_url : String, priority : String, changefreq : String do
+    ECR.def_to_s "#{__DIR__}/html/sitemap.xml"
+  end
 end