From 6bea6bee790673592da2d1af784b79339bb8c2c6 Mon Sep 17 00:00:00 2001 From: Dimitar Kostadinov Date: Wed, 8 Dec 2021 17:09:39 +0200 Subject: [PATCH 1/6] Integrate Goldmark Markdown parser --- cmd/app/cmd.go | 13 +- cmd/app/cmd_test.go | 110 - cmd/app/factory.go | 58 +- go.mod | 7 +- go.sum | 4 + .../mainTree/html-tests/file1.md | 1 + .../nodeSelector/innerDir/testedHTMLFile5.md | 1 + .../nodeSelector/testedHTMLFile3.md | 1 + .../nodeSelector/testedHTMLFile4.md | 1 + .../mainTree/html-tests/testedHTMLFile2.md | 1 + .../mainTree/markdown-tests/file1.md | 1 + .../innerDir/testedMarkdownFile5.md | 3 +- .../nodeSelector/testedMarkdownFile3.md | 3 +- .../nodeSelector/testedMarkdownFile4.md | 3 +- .../markdown-tests/testedMarkdownFile2.md | 3 +- .../level1-container-node1/fileD.md | 1 + .../level2-container-node1/fileD.md | 1 + .../level3-container-node1/file1.md | 1 + .../level3-container-node1/file2.md | 1 + .../level3-container-node1/fileD1.md | 1 + .../level3-container-node1/fileD2.md | 1 + .../level3-container-node1/fileY1.md | 1 + .../level3-container-node1/fileY2.md | 1 + .../level2-container-node2/file1.md | 1 + .../level2-container-node2/fileD.md | 1 + .../level2-container-node2/fileY.md | 1 + .../level1-container-node2/file1.md | 1 + .../level1-container-node2/fileD.md | 1 + .../level1-container-node2/fileY.md | 1 + pkg/hugo/fswriter.go | 80 - pkg/hugo/fswriter_test.go | 113 - pkg/hugo/hugo.go | 45 - pkg/hugo/processor.go | 161 - pkg/hugo/processor_test.go | 295 - pkg/markdown/frontmatter.go | 199 - pkg/markdown/frontmatter_test.go | 223 - pkg/markdown/html_links.go | 74 - pkg/markdown/link_modifier.go | 987 ++++ pkg/markdown/link_modifier_test.go | 220 + pkg/markdown/links.go | 103 - pkg/markdown/links_test.go | 68 - pkg/markdown/markdown_suite_test.go | 16 + pkg/markdown/parser.go | 39 + pkg/markdown/parser/autolinks_parse.go | 475 -- pkg/markdown/parser/autolinks_parse_test.go | 244 - pkg/markdown/parser/charscan.go | 87 - pkg/markdown/parser/document.go | 37 - pkg/markdown/parser/links.go | 177 - pkg/markdown/parser/links_parse.go | 486 -- pkg/markdown/parser/links_parse_test.go | 300 - pkg/markdown/parser/links_test.go | 379 -- pkg/markdown/parser/parser.go | 68 - pkg/markdown/parser/parser_test.go | 93 - pkg/markdown/parser/types.go | 41 - pkg/markdown/parser_test.go | 68 + pkg/markdown/spec/commonmark_v.0.30.json | 5218 +++++++++++++++++ pkg/markdown/spec/gfm_v.0.29.json | 148 + pkg/markdown/specs_test.go | 155 + pkg/processors/document.go | 65 - pkg/processors/frontmatter.go | 115 - pkg/processors/frontmatter_test.go | 88 - pkg/processors/processor.go | 27 - pkg/reactor/content_processor.go | 781 ++- pkg/reactor/content_processor_test.go | 356 +- pkg/reactor/document_worker.go | 153 +- pkg/reactor/integration_test.go | 127 - pkg/reactor/reactor.go | 40 +- .../git/gitGinko_resource_handler_test.go | 6 +- .../github/github_resource_handler.go | 18 +- pkg/writers/fswriter.go | 36 +- .../github.com/yuin/goldmark-meta/.gitignore | 13 + vendor/github.com/yuin/goldmark-meta/LICENSE | 21 + .../github.com/yuin/goldmark-meta/README.md | 152 + vendor/github.com/yuin/goldmark-meta/meta.go | 248 + vendor/github.com/yuin/goldmark/.gitignore | 19 + vendor/github.com/yuin/goldmark/LICENSE | 21 + vendor/github.com/yuin/goldmark/Makefile | 16 + vendor/github.com/yuin/goldmark/README.md | 494 ++ vendor/github.com/yuin/goldmark/ast/ast.go | 508 ++ vendor/github.com/yuin/goldmark/ast/block.go | 495 ++ vendor/github.com/yuin/goldmark/ast/inline.go | 548 ++ .../goldmark/extension/ast/definition_list.go | 83 + .../yuin/goldmark/extension/ast/footnote.go | 132 + .../goldmark/extension/ast/strikethrough.go | 29 + .../yuin/goldmark/extension/ast/table.go | 157 + .../yuin/goldmark/extension/ast/tasklist.go | 35 + .../goldmark/extension/definition_list.go | 270 + .../yuin/goldmark/extension/footnote.go | 670 +++ .../github.com/yuin/goldmark/extension/gfm.go | 18 + .../yuin/goldmark/extension/linkify.go | 318 + .../yuin/goldmark/extension/strikethrough.go | 116 + .../yuin/goldmark/extension/table.go | 552 ++ .../yuin/goldmark/extension/tasklist.go | 115 + .../yuin/goldmark/extension/typographer.go | 323 + vendor/github.com/yuin/goldmark/markdown.go | 140 + .../yuin/goldmark/parser/attribute.go | 328 ++ .../yuin/goldmark/parser/atx_heading.go | 246 + .../yuin/goldmark/parser/auto_link.go | 42 + .../yuin/goldmark/parser/blockquote.go | 69 + .../yuin/goldmark/parser/code_block.go | 100 + .../yuin/goldmark/parser/code_span.go | 84 + .../yuin/goldmark/parser/delimiter.go | 238 + .../yuin/goldmark/parser/emphasis.go | 50 + .../yuin/goldmark/parser/fcode_block.go | 121 + .../yuin/goldmark/parser/html_block.go | 228 + .../github.com/yuin/goldmark/parser/link.go | 382 ++ .../yuin/goldmark/parser/link_ref.go | 152 + .../github.com/yuin/goldmark/parser/list.go | 283 + .../yuin/goldmark/parser/list_item.go | 90 + .../yuin/goldmark/parser/paragraph.go | 71 + .../github.com/yuin/goldmark/parser/parser.go | 1236 ++++ .../yuin/goldmark/parser/raw_html.go | 104 + .../yuin/goldmark/parser/setext_headings.go | 126 + .../yuin/goldmark/parser/thematic_break.go | 75 + .../yuin/goldmark/renderer/html/html.go | 831 +++ .../yuin/goldmark/renderer/renderer.go | 174 + .../github.com/yuin/goldmark/text/reader.go | 653 +++ .../github.com/yuin/goldmark/text/segment.go | 209 + .../yuin/goldmark/util/html5entities.go | 2142 +++++++ .../goldmark/util/unicode_case_folding.go | 1491 +++++ vendor/github.com/yuin/goldmark/util/util.go | 980 ++++ .../yuin/goldmark/util/util_safe.go | 13 + .../yuin/goldmark/util/util_unsafe.go | 23 + vendor/modules.txt | 14 + 124 files changed, 23699 insertions(+), 4984 deletions(-) delete mode 100644 pkg/hugo/fswriter.go delete mode 100644 pkg/hugo/fswriter_test.go delete mode 100644 pkg/hugo/hugo.go delete mode 100755 pkg/hugo/processor.go delete mode 100755 pkg/hugo/processor_test.go delete mode 100644 pkg/markdown/frontmatter.go delete mode 100644 pkg/markdown/frontmatter_test.go delete mode 100644 pkg/markdown/html_links.go create mode 100644 pkg/markdown/link_modifier.go create mode 100644 pkg/markdown/link_modifier_test.go delete mode 100644 pkg/markdown/links.go delete mode 100644 pkg/markdown/links_test.go create mode 100644 pkg/markdown/markdown_suite_test.go create mode 100644 pkg/markdown/parser.go delete mode 100644 pkg/markdown/parser/autolinks_parse.go delete mode 100644 pkg/markdown/parser/autolinks_parse_test.go delete mode 100644 pkg/markdown/parser/charscan.go delete mode 100644 pkg/markdown/parser/document.go delete mode 100644 pkg/markdown/parser/links.go delete mode 100644 pkg/markdown/parser/links_parse.go delete mode 100644 pkg/markdown/parser/links_parse_test.go delete mode 100644 pkg/markdown/parser/links_test.go delete mode 100644 pkg/markdown/parser/parser.go delete mode 100644 pkg/markdown/parser/parser_test.go delete mode 100644 pkg/markdown/parser/types.go create mode 100644 pkg/markdown/parser_test.go create mode 100644 pkg/markdown/spec/commonmark_v.0.30.json create mode 100644 pkg/markdown/spec/gfm_v.0.29.json create mode 100644 pkg/markdown/specs_test.go delete mode 100644 pkg/processors/document.go delete mode 100644 pkg/processors/frontmatter.go delete mode 100644 pkg/processors/frontmatter_test.go delete mode 100644 pkg/processors/processor.go delete mode 100644 pkg/reactor/integration_test.go create mode 100644 vendor/github.com/yuin/goldmark-meta/.gitignore create mode 100644 vendor/github.com/yuin/goldmark-meta/LICENSE create mode 100644 vendor/github.com/yuin/goldmark-meta/README.md create mode 100644 vendor/github.com/yuin/goldmark-meta/meta.go create mode 100644 vendor/github.com/yuin/goldmark/.gitignore create mode 100644 vendor/github.com/yuin/goldmark/LICENSE create mode 100644 vendor/github.com/yuin/goldmark/Makefile create mode 100644 vendor/github.com/yuin/goldmark/README.md create mode 100644 vendor/github.com/yuin/goldmark/ast/ast.go create mode 100644 vendor/github.com/yuin/goldmark/ast/block.go create mode 100644 vendor/github.com/yuin/goldmark/ast/inline.go create mode 100644 vendor/github.com/yuin/goldmark/extension/ast/definition_list.go create mode 100644 vendor/github.com/yuin/goldmark/extension/ast/footnote.go create mode 100644 vendor/github.com/yuin/goldmark/extension/ast/strikethrough.go create mode 100644 vendor/github.com/yuin/goldmark/extension/ast/table.go create mode 100644 vendor/github.com/yuin/goldmark/extension/ast/tasklist.go create mode 100644 vendor/github.com/yuin/goldmark/extension/definition_list.go create mode 100644 vendor/github.com/yuin/goldmark/extension/footnote.go create mode 100644 vendor/github.com/yuin/goldmark/extension/gfm.go create mode 100644 vendor/github.com/yuin/goldmark/extension/linkify.go create mode 100644 vendor/github.com/yuin/goldmark/extension/strikethrough.go create mode 100644 vendor/github.com/yuin/goldmark/extension/table.go create mode 100644 vendor/github.com/yuin/goldmark/extension/tasklist.go create mode 100644 vendor/github.com/yuin/goldmark/extension/typographer.go create mode 100644 vendor/github.com/yuin/goldmark/markdown.go create mode 100644 vendor/github.com/yuin/goldmark/parser/attribute.go create mode 100644 vendor/github.com/yuin/goldmark/parser/atx_heading.go create mode 100644 vendor/github.com/yuin/goldmark/parser/auto_link.go create mode 100644 vendor/github.com/yuin/goldmark/parser/blockquote.go create mode 100644 vendor/github.com/yuin/goldmark/parser/code_block.go create mode 100644 vendor/github.com/yuin/goldmark/parser/code_span.go create mode 100644 vendor/github.com/yuin/goldmark/parser/delimiter.go create mode 100644 vendor/github.com/yuin/goldmark/parser/emphasis.go create mode 100644 vendor/github.com/yuin/goldmark/parser/fcode_block.go create mode 100644 vendor/github.com/yuin/goldmark/parser/html_block.go create mode 100644 vendor/github.com/yuin/goldmark/parser/link.go create mode 100644 vendor/github.com/yuin/goldmark/parser/link_ref.go create mode 100644 vendor/github.com/yuin/goldmark/parser/list.go create mode 100644 vendor/github.com/yuin/goldmark/parser/list_item.go create mode 100644 vendor/github.com/yuin/goldmark/parser/paragraph.go create mode 100644 vendor/github.com/yuin/goldmark/parser/parser.go create mode 100644 vendor/github.com/yuin/goldmark/parser/raw_html.go create mode 100644 vendor/github.com/yuin/goldmark/parser/setext_headings.go create mode 100644 vendor/github.com/yuin/goldmark/parser/thematic_break.go create mode 100644 vendor/github.com/yuin/goldmark/renderer/html/html.go create mode 100644 vendor/github.com/yuin/goldmark/renderer/renderer.go create mode 100644 vendor/github.com/yuin/goldmark/text/reader.go create mode 100644 vendor/github.com/yuin/goldmark/text/segment.go create mode 100644 vendor/github.com/yuin/goldmark/util/html5entities.go create mode 100644 vendor/github.com/yuin/goldmark/util/unicode_case_folding.go create mode 100644 vendor/github.com/yuin/goldmark/util/util.go create mode 100644 vendor/github.com/yuin/goldmark/util/util_safe.go create mode 100644 vendor/github.com/yuin/goldmark/util/util_unsafe.go diff --git a/cmd/app/cmd.go b/cmd/app/cmd.go index 509b24f3..265bc67d 100644 --- a/cmd/app/cmd.go +++ b/cmd/app/cmd.go @@ -14,7 +14,6 @@ import ( "github.com/gardener/docforge/cmd/configuration" "github.com/gardener/docforge/pkg/api" - "github.com/gardener/docforge/pkg/hugo" "github.com/gardener/docforge/pkg/resourcehandlers" "github.com/spf13/cobra" "k8s.io/klog/v2" @@ -310,27 +309,27 @@ func cacheHomeDir(f *cmdFlags, config *configuration.Config) string { return filepath.Join(userHomeDir, configuration.DocforgeHomeDir) } -func hugoOptions(f *cmdFlags, config *configuration.Config) *hugo.Options { +func hugoOptions(f *cmdFlags, config *configuration.Config) *Hugo { if !f.hugo && (config == nil || config.Hugo == nil) { return nil } - hugoOptions := &hugo.Options{ + ho := &Hugo{ PrettyUrls: prettyURLs(f.hugoPrettyUrls, config.Hugo), IndexFileNames: combineSectionFiles(f.hugoSectionFiles, config.Hugo), } if f.hugoBaseURL != "" { - hugoOptions.BaseURL = f.hugoBaseURL - return hugoOptions + ho.BaseURL = f.hugoBaseURL + return ho } if config.Hugo != nil { if config.Hugo.BaseURL != nil { - hugoOptions.BaseURL = *config.Hugo.BaseURL + ho.BaseURL = *config.Hugo.BaseURL } } - return hugoOptions + return ho } func combineSectionFiles(sectionFilesFromFlags []string, hugoConfig *configuration.Hugo) []string { diff --git a/cmd/app/cmd_test.go b/cmd/app/cmd_test.go index 0c9ccf26..15f1d8ff 100644 --- a/cmd/app/cmd_test.go +++ b/cmd/app/cmd_test.go @@ -5,123 +5,13 @@ package app import ( - "reflect" "testing" "github.com/gardener/docforge/cmd/configuration" - "github.com/gardener/docforge/pkg/hugo" "github.com/stretchr/testify/assert" "k8s.io/utils/pointer" ) -func Test_hugoOptions(t *testing.T) { - type args struct { - f *cmdFlags - config *configuration.Config - } - tests := []struct { - name string - args args - want *hugo.Options - }{ - { - name: "return_nil_when_no_config_or_flag_provided", - args: args{ - f: &cmdFlags{ - hugo: false, - }, - config: &configuration.Config{ - Hugo: nil, - }, - }, - want: nil, - }, - { - name: "return_default_hugo_options", - args: args{ - f: &cmdFlags{ - hugo: true, - hugoPrettyUrls: true, - }, - config: &configuration.Config{ - Hugo: nil, - }, - }, - want: &hugo.Options{ - PrettyUrls: true, - IndexFileNames: []string{}, - BaseURL: "", - }, - }, - { - name: "use_base_url_from_config_when_not_specified_in_flags", - args: args{ - f: &cmdFlags{ - hugo: true, - hugoPrettyUrls: true, - }, - config: &configuration.Config{ - Hugo: &configuration.Hugo{ - BaseURL: pointer.StringPtr("/new/baseURL"), - }, - }, - }, - want: &hugo.Options{ - PrettyUrls: true, - IndexFileNames: []string{}, - BaseURL: "/new/baseURL", - }, - }, - { - name: "use_base_url_from_flags_with_priority", - args: args{ - f: &cmdFlags{ - hugo: true, - hugoPrettyUrls: true, - hugoBaseURL: "/override", - }, - config: &configuration.Config{ - Hugo: &configuration.Hugo{ - BaseURL: pointer.StringPtr("/new/baseURL"), - }, - }, - }, - want: &hugo.Options{ - PrettyUrls: true, - IndexFileNames: []string{}, - BaseURL: "/override", - }, - }, - { - name: "set_hugo_base_url_from_flags", - args: args{ - f: &cmdFlags{ - hugo: true, - hugoPrettyUrls: true, - hugoBaseURL: "/fromFlag", - }, - config: &configuration.Config{ - Hugo: &configuration.Hugo{ - BaseURL: nil, - }, - }, - }, - want: &hugo.Options{ - PrettyUrls: true, - IndexFileNames: []string{}, - BaseURL: "/fromFlag", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := hugoOptions(tt.args.f, tt.args.config); !reflect.DeepEqual(got, tt.want) { - t.Errorf("hugoOptions() = %v, want %v", got, tt.want) - } - }) - } -} - func Test_prettyURLs(t *testing.T) { type args struct { fromFlag bool diff --git a/cmd/app/factory.go b/cmd/app/factory.go index 5a205cfb..1e23ed00 100644 --- a/cmd/app/factory.go +++ b/cmd/app/factory.go @@ -18,15 +18,12 @@ import ( "k8s.io/klog/v2" "github.com/gardener/docforge/pkg/api" - "github.com/gardener/docforge/pkg/hugo" - "github.com/gardener/docforge/pkg/resourcehandlers" - "github.com/gardener/docforge/pkg/writers" - "github.com/hashicorp/go-multierror" - - "github.com/gardener/docforge/pkg/processors" "github.com/gardener/docforge/pkg/reactor" + "github.com/gardener/docforge/pkg/resourcehandlers" "github.com/gardener/docforge/pkg/resourcehandlers/git" ghrs "github.com/gardener/docforge/pkg/resourcehandlers/github" + "github.com/gardener/docforge/pkg/writers" + "github.com/hashicorp/go-multierror" "github.com/google/go-github/v32/github" "golang.org/x/oauth2" @@ -56,7 +53,7 @@ type Options struct { GitHubInfoPath string DryRunWriter io.Writer Resolve bool - Hugo *hugo.Options + Hugo *Hugo UseGit bool HomeDir string LocalMappings map[string]string @@ -64,6 +61,22 @@ type Options struct { LastNVersions map[string]int } +// Hugo is the configuration options for creating HUGO implementations +// docforge interfaces +type Hugo struct { + // PrettyUrls indicates if links will rewritten for Hugo will be + // formatted for pretty url support or not. Pretty urls in Hugo + // place built source content in index.html, which resides in a path segment with + // the name of the file, making request URLs more resource-oriented. + // Example: (source) sample.md -> (build) sample/index.html -> (runtime) ./sample + PrettyUrls bool + // IndexFileNames defines a list of file names that indicate + // their content can be used as Hugo section files (_index.md). + IndexFileNames []string + // BaseURL is used from the Hugo processor to rewrite relative links to root-relative + BaseURL string +} + // Credentials holds repositories access credentials type Credentials struct { Host string @@ -89,7 +102,6 @@ func NewReactor(ctx context.Context, options *Options, rhs []resourcehandlers.Re ResourcesPath: options.ResourcesPath, ResourceDownloadWorkersCount: options.ResourceDownloadWorkersCount, RewriteEmbedded: options.RewriteEmbedded, - Processor: nil, ResourceHandlers: rhs, DryRunWriter: dryRunWriters, Resolve: options.Resolve, @@ -103,6 +115,7 @@ func NewReactor(ctx context.Context, options *Options, rhs []resourcehandlers.Re } else { o.Writer = &writers.FSWriter{ Root: options.DestinationPath, + Hugo: options.Hugo != nil, } o.ResourceDownloadWriter = &writers.FSWriter{ Root: filepath.Join(options.DestinationPath, options.ResourcesPath), @@ -117,32 +130,13 @@ func NewReactor(ctx context.Context, options *Options, rhs []resourcehandlers.Re } if options.Hugo != nil { - WithHugo(o, options) + o.Hugo = true + o.PrettyUrls = options.Hugo.PrettyUrls + o.IndexFileNames = options.Hugo.IndexFileNames + o.BaseURL = options.Hugo.BaseURL } - return reactor.NewReactor(o) -} -// WithHugo adapts the reactor.Options object with Hugo-specific -// settings for writer and processor -func WithHugo(reactorOptions *reactor.Options, o *Options) { - hugoOptions := o.Hugo - reactorOptions.Processor = &processors.ProcessorChain{ - Processors: []processors.Processor{ - &processors.FrontMatter{ - IndexFileNames: hugoOptions.IndexFileNames, - }, - hugo.NewProcessor(hugoOptions), - }, - } - if o.DryRunWriter != nil { - hugoOptions.Writer = reactorOptions.Writer - } else { - hugoOptions.Writer = &writers.FSWriter{ - Root: filepath.Join(o.DestinationPath), - } - } - reactorOptions.IndexFileNames = hugoOptions.IndexFileNames - reactorOptions.Writer = hugo.NewWriter(hugoOptions) + return reactor.NewReactor(o) } func initResourceHandlers(ctx context.Context, o *Options) ([]resourcehandlers.ResourceHandler, error) { diff --git a/go.mod b/go.mod index 273f1863..22631c27 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,12 @@ require ( k8s.io/utils v0.0.0-20201027101359-01387209bb0d ) +require ( + github.com/yuin/goldmark v1.4.4 + github.com/yuin/goldmark-meta v1.0.0 + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d +) + require ( github.com/Microsoft/go-winio v0.4.16 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -59,7 +65,6 @@ require ( github.com/xanzy/ssh-agent v0.3.0 // indirect golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect golang.org/x/mod v0.4.2 // indirect - golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.1 // indirect diff --git a/go.sum b/go.sum index 033d315a..f43883dc 100644 --- a/go.sum +++ b/go.sum @@ -296,6 +296,10 @@ github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6e github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.4 h1:zNWRjYUW32G9KirMXYHQHVNFkXvMI7LpgNW2AgYAoIs= +github.com/yuin/goldmark v1.4.4/go.mod h1:rmuwmfZ0+bvzB24eSC//bk1R1Zp3hM0OXYv/G2LIilg= +github.com/yuin/goldmark-meta v1.0.0 h1:ScsatUIT2gFS6azqzLGUjgOnELsBOxMXerM3ogdJhAM= +github.com/yuin/goldmark-meta v1.0.0/go.mod h1:zsNNOrZ4nLuyHAJeLQEZcQat8dm70SmB2kHbls092Gc= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= diff --git a/integration-test/expected-tree/mainTree/html-tests/file1.md b/integration-test/expected-tree/mainTree/html-tests/file1.md index 5055b52e..f175b671 100644 --- a/integration-test/expected-tree/mainTree/html-tests/file1.md +++ b/integration-test/expected-tree/mainTree/html-tests/file1.md @@ -1,6 +1,7 @@ --- title: File1 --- +

Tested HTML file 1

Link file which is not in the structure

diff --git a/integration-test/expected-tree/mainTree/html-tests/nodeSelector/innerDir/testedHTMLFile5.md b/integration-test/expected-tree/mainTree/html-tests/nodeSelector/innerDir/testedHTMLFile5.md index 75eb0d27..904d91d8 100644 --- a/integration-test/expected-tree/mainTree/html-tests/nodeSelector/innerDir/testedHTMLFile5.md +++ b/integration-test/expected-tree/mainTree/html-tests/nodeSelector/innerDir/testedHTMLFile5.md @@ -1,6 +1,7 @@ --- title: TestedHTMLFile5 --- +

Tested HTML file 5

Link relatively file which is in the structure

diff --git a/integration-test/expected-tree/mainTree/html-tests/nodeSelector/testedHTMLFile3.md b/integration-test/expected-tree/mainTree/html-tests/nodeSelector/testedHTMLFile3.md index 0b66e795..f0a7459a 100644 --- a/integration-test/expected-tree/mainTree/html-tests/nodeSelector/testedHTMLFile3.md +++ b/integration-test/expected-tree/mainTree/html-tests/nodeSelector/testedHTMLFile3.md @@ -1,6 +1,7 @@ --- title: TestedHTMLFile3 --- +

Tested HTML file 3

Link relatively file which is in the structure

diff --git a/integration-test/expected-tree/mainTree/html-tests/nodeSelector/testedHTMLFile4.md b/integration-test/expected-tree/mainTree/html-tests/nodeSelector/testedHTMLFile4.md index fea5d86c..50af39f6 100644 --- a/integration-test/expected-tree/mainTree/html-tests/nodeSelector/testedHTMLFile4.md +++ b/integration-test/expected-tree/mainTree/html-tests/nodeSelector/testedHTMLFile4.md @@ -1,6 +1,7 @@ --- title: TestedHTMLFile4 --- +

Tested HTML file 4

Link file which is not in the structure

diff --git a/integration-test/expected-tree/mainTree/html-tests/testedHTMLFile2.md b/integration-test/expected-tree/mainTree/html-tests/testedHTMLFile2.md index f5a1df03..c9eb370f 100644 --- a/integration-test/expected-tree/mainTree/html-tests/testedHTMLFile2.md +++ b/integration-test/expected-tree/mainTree/html-tests/testedHTMLFile2.md @@ -1,6 +1,7 @@ --- title: TestedHTMLFile2 --- +

Tested HTML file 2

Link file which is not in the structure

diff --git a/integration-test/expected-tree/mainTree/markdown-tests/file1.md b/integration-test/expected-tree/mainTree/markdown-tests/file1.md index 73c9d6ea..38141732 100644 --- a/integration-test/expected-tree/mainTree/markdown-tests/file1.md +++ b/integration-test/expected-tree/mainTree/markdown-tests/file1.md @@ -1,6 +1,7 @@ --- title: testedFile1 --- + # Tested markdown file 1 ### Link file which is not in the structure diff --git a/integration-test/expected-tree/mainTree/markdown-tests/nodeSelector/innerDir/testedMarkdownFile5.md b/integration-test/expected-tree/mainTree/markdown-tests/nodeSelector/innerDir/testedMarkdownFile5.md index 9b90f2e9..f3ba6337 100644 --- a/integration-test/expected-tree/mainTree/markdown-tests/nodeSelector/innerDir/testedMarkdownFile5.md +++ b/integration-test/expected-tree/mainTree/markdown-tests/nodeSelector/innerDir/testedMarkdownFile5.md @@ -1,6 +1,7 @@ --- title: TestedMarkdownFile5 --- + # Tested markdown file 5 ### Link relatively file which is in the structure @@ -13,4 +14,4 @@ title: TestedMarkdownFile5 ![test3](/__resources/gardener-docforge-logo.png) ### Link existing image with relative path and title -![test4](/__resources/gardener-docforge-logo.png "gardener-docforge-logo") \ No newline at end of file +![test4](/__resources/gardener-docforge-logo.png "gardener-docforge-logo") diff --git a/integration-test/expected-tree/mainTree/markdown-tests/nodeSelector/testedMarkdownFile3.md b/integration-test/expected-tree/mainTree/markdown-tests/nodeSelector/testedMarkdownFile3.md index 5990f0ec..3af39c11 100644 --- a/integration-test/expected-tree/mainTree/markdown-tests/nodeSelector/testedMarkdownFile3.md +++ b/integration-test/expected-tree/mainTree/markdown-tests/nodeSelector/testedMarkdownFile3.md @@ -1,6 +1,7 @@ --- title: TestedMarkdownFile3 --- + # Tested markdown file 3 ### Link relatively file which is in the structure @@ -16,4 +17,4 @@ title: TestedMarkdownFile3 ![test4](/__resources/gardener-docforge-logo.png) ### Link existing image with relative path and title -![test5](/__resources/gardener-docforge-logo.png "gardener-docforge-logo") \ No newline at end of file +![test5](/__resources/gardener-docforge-logo.png "gardener-docforge-logo") diff --git a/integration-test/expected-tree/mainTree/markdown-tests/nodeSelector/testedMarkdownFile4.md b/integration-test/expected-tree/mainTree/markdown-tests/nodeSelector/testedMarkdownFile4.md index bc31fef4..3c6fca8d 100644 --- a/integration-test/expected-tree/mainTree/markdown-tests/nodeSelector/testedMarkdownFile4.md +++ b/integration-test/expected-tree/mainTree/markdown-tests/nodeSelector/testedMarkdownFile4.md @@ -1,6 +1,7 @@ --- title: TestedMarkdownFile4 --- + # Tested markdown file 4 ### Link file which is not in the structure @@ -10,4 +11,4 @@ title: TestedMarkdownFile4 ![test2](/__resources/gardener-docforge-logo.png) ### Link existing image with relative path and title -![test3](/__resources/gardener-docforge-logo.png "gardener-docforge-logo") \ No newline at end of file +![test3](/__resources/gardener-docforge-logo.png "gardener-docforge-logo") diff --git a/integration-test/expected-tree/mainTree/markdown-tests/testedMarkdownFile2.md b/integration-test/expected-tree/mainTree/markdown-tests/testedMarkdownFile2.md index 86a8a957..7776898f 100644 --- a/integration-test/expected-tree/mainTree/markdown-tests/testedMarkdownFile2.md +++ b/integration-test/expected-tree/mainTree/markdown-tests/testedMarkdownFile2.md @@ -1,6 +1,7 @@ --- title: TestedMarkdownFile2 --- + # Tested markdown file 2 ### Link file which is not in the structure @@ -13,4 +14,4 @@ title: TestedMarkdownFile2 ![test3](/__resources/gardener-docforge-logo.png) ### Link existing image with relative path and title -![test4](/__resources/gardener-docforge-logo.png "gardener-docforge-logo") \ No newline at end of file +![test4](/__resources/gardener-docforge-logo.png "gardener-docforge-logo") diff --git a/integration-test/expected-tree/merge-node/level1-container-node1/fileD.md b/integration-test/expected-tree/merge-node/level1-container-node1/fileD.md index cb6f2001..c6ce6ff3 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node1/fileD.md +++ b/integration-test/expected-tree/merge-node/level1-container-node1/fileD.md @@ -1,4 +1,5 @@ --- title: FileD --- + # Content diff --git a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/fileD.md b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/fileD.md index cb6f2001..c6ce6ff3 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/fileD.md +++ b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/fileD.md @@ -1,4 +1,5 @@ --- title: FileD --- + # Content diff --git a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/file1.md b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/file1.md index f64850cd..9695192e 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/file1.md +++ b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/file1.md @@ -1,4 +1,5 @@ --- title: File1 --- + # Content diff --git a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/file2.md b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/file2.md index ccd34ecf..c173158f 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/file2.md +++ b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/file2.md @@ -1,4 +1,5 @@ --- title: File2 --- + # Content diff --git a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileD1.md b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileD1.md index 27e060b6..1bcf4027 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileD1.md +++ b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileD1.md @@ -1,4 +1,5 @@ --- title: FileD1 --- + # Content diff --git a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileD2.md b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileD2.md index 1b6a6304..739b1228 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileD2.md +++ b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileD2.md @@ -1,4 +1,5 @@ --- title: FileD2 --- + # Content diff --git a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileY1.md b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileY1.md index f0f915dd..07240b33 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileY1.md +++ b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileY1.md @@ -1,4 +1,5 @@ --- title: FileY1 --- + # Content diff --git a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileY2.md b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileY2.md index 96ef7a5f..89208d4c 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileY2.md +++ b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node1/level3-container-node1/fileY2.md @@ -1,4 +1,5 @@ --- title: FileY2 --- + # Content diff --git a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node2/file1.md b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node2/file1.md index f64850cd..9695192e 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node2/file1.md +++ b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node2/file1.md @@ -1,4 +1,5 @@ --- title: File1 --- + # Content diff --git a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node2/fileD.md b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node2/fileD.md index cb6f2001..c6ce6ff3 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node2/fileD.md +++ b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node2/fileD.md @@ -1,4 +1,5 @@ --- title: FileD --- + # Content diff --git a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node2/fileY.md b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node2/fileY.md index f83b9bdd..914e6235 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node2/fileY.md +++ b/integration-test/expected-tree/merge-node/level1-container-node1/level2-container-node2/fileY.md @@ -1,4 +1,5 @@ --- title: FileY --- + # Content diff --git a/integration-test/expected-tree/merge-node/level1-container-node2/file1.md b/integration-test/expected-tree/merge-node/level1-container-node2/file1.md index f64850cd..9695192e 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node2/file1.md +++ b/integration-test/expected-tree/merge-node/level1-container-node2/file1.md @@ -1,4 +1,5 @@ --- title: File1 --- + # Content diff --git a/integration-test/expected-tree/merge-node/level1-container-node2/fileD.md b/integration-test/expected-tree/merge-node/level1-container-node2/fileD.md index cb6f2001..c6ce6ff3 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node2/fileD.md +++ b/integration-test/expected-tree/merge-node/level1-container-node2/fileD.md @@ -1,4 +1,5 @@ --- title: FileD --- + # Content diff --git a/integration-test/expected-tree/merge-node/level1-container-node2/fileY.md b/integration-test/expected-tree/merge-node/level1-container-node2/fileY.md index f83b9bdd..914e6235 100644 --- a/integration-test/expected-tree/merge-node/level1-container-node2/fileY.md +++ b/integration-test/expected-tree/merge-node/level1-container-node2/fileY.md @@ -1,4 +1,5 @@ --- title: FileY --- + # Content diff --git a/pkg/hugo/fswriter.go b/pkg/hugo/fswriter.go deleted file mode 100644 index a86d2cdc..00000000 --- a/pkg/hugo/fswriter.go +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package hugo - -import ( - "fmt" - "github.com/gardener/docforge/pkg/api" - "github.com/gardener/docforge/pkg/markdown" - "github.com/gardener/docforge/pkg/writers" - "gopkg.in/yaml.v3" - "k8s.io/klog/v2" - "path/filepath" -) - -// FSWriter is implementation of Writer interface for writing blobs -// to the file system at a designated path in a Hugo-specific way -type FSWriter struct { - Writer writers.Writer -} - -// Write implements writers#Write and will create section file #path/#name/_index.md' -// for node without content, but with frontmatter properties. -func (w *FSWriter) Write(name, path string, docBlob []byte, node *api.Node) error { - if node != nil { - if docBlob == nil && node.Properties != nil && node.Properties["frontmatter"] != nil { - if len(node.Nodes) > 0 { - // this is a container node -> propagate frontmatter properties to child _index.md if any - for _, n := range node.Nodes { - if n.Name == "_index.md" { - klog.V(6).Infof("merge %s/_index.md frontmatter properties with: %v\n", api.Path(n, "/"), node.Properties["frontmatter"]) - if n.Properties == nil { - n.Properties = map[string]interface{}{"frontmatter": node.Properties["frontmatter"]} - } else { - childFrontmatter := n.Properties["frontmatter"] - if childFrontmatter == nil { - n.Properties["frontmatter"] = node.Properties["frontmatter"] - } else { - mergedFrontmatter, ok := childFrontmatter.(map[string]interface{}) - if !ok { - return fmt.Errorf("invalid frontmatter properties for node: %s/%s", api.Path(n, "/"), n.Name) - } - parentFrontmatter, ok := node.Properties["frontmatter"].(map[string]interface{}) - if !ok { - return fmt.Errorf("invalid frontmatter properties for node: %s/%s", api.Path(node, "/"), node.Name) - } - for k, v := range parentFrontmatter { - mergedFrontmatter[k] = v - } - } - } - klog.V(6).Infof("merged properties: %v", n.Properties["frontmatter"]) - return nil - } - } - } - var ( - err error - b []byte - _docBlob []byte - ) - if b, err = yaml.Marshal(node.Properties["frontmatter"]); err != nil { - return err - } - _name := "_index.md" - if _docBlob, err = markdown.InsertFrontMatter(b, []byte("")); err != nil { - return err - } - if err := w.Writer.Write(_name, filepath.Join(path, name), _docBlob, node); err != nil { - return err - } - } - } - - if err := w.Writer.Write(name, path, docBlob, node); err != nil { - return err - } - return nil -} diff --git a/pkg/hugo/fswriter_test.go b/pkg/hugo/fswriter_test.go deleted file mode 100644 index 4918119d..00000000 --- a/pkg/hugo/fswriter_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package hugo - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "testing" - - "github.com/gardener/docforge/pkg/api" - "github.com/gardener/docforge/pkg/util/tests" - "github.com/gardener/docforge/pkg/writers" - "github.com/google/uuid" -) - -func init() { - tests.SetKlogV(6) -} - -func TestWrite(t *testing.T) { - testCases := []struct { - name string - path string - docBlob []byte - node *api.Node - wantErr error - wantFileName string - wantContent string - mutate func(writer *FSWriter) - }{ - { - name: "test.md", - path: "a/b", - docBlob: []byte("# Test"), - wantErr: nil, - wantFileName: `test.md`, - wantContent: `# Test`, - }, - { - name: "test", - path: "a/b", - docBlob: []byte("# Test"), - node: &api.Node{}, - wantErr: nil, - wantFileName: `test`, - wantContent: `# Test`, - }, - { - name: "test", - path: "a/b", - docBlob: nil, - node: &api.Node{ - Properties: map[string]interface{}{ - "frontmatter": map[string]string{ - "title": "Test1", - }, - }, - }, - wantErr: nil, - wantFileName: filepath.Join("test", "_index.md"), - wantContent: `--- -title: Test1 ---- -`, - }, - } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - testFolder := fmt.Sprintf("test%s", uuid.New().String()) - testPath := filepath.Join(os.TempDir(), testFolder) - fs := &FSWriter{ - Writer: &writers.FSWriter{ - Root: testPath, - }, - } - if tc.mutate != nil { - tc.mutate(fs) - } - fPath := filepath.Join(testPath, tc.path, tc.wantFileName) - defer func() { - if err := os.RemoveAll(testPath); err != nil { - t.Fatalf("%v\n", err) - } - }() - - if tc.node != nil { - tc.node.SetParentsDownwards() - } - err := fs.Write(tc.name, tc.path, tc.docBlob, tc.node) - - if err != tc.wantErr { - t.Errorf("expected err %v != %v", tc.wantErr, err) - } - if _, err := os.Stat(fPath); tc.wantErr == nil && os.IsNotExist(err) { - t.Errorf("expected file %s not found: %v", fPath, err) - } - var ( - gotContent []byte - ) - if gotContent, err = ioutil.ReadFile(fPath); err != nil { - t.Errorf("unexpected error opening file %v", err) - } - if !reflect.DeepEqual(gotContent, []byte(tc.wantContent)) { - t.Errorf("expected content \n%v\n != \n%v\n", tc.wantContent, string(gotContent)) - } - }) - } -} diff --git a/pkg/hugo/hugo.go b/pkg/hugo/hugo.go deleted file mode 100644 index cbd6f22e..00000000 --- a/pkg/hugo/hugo.go +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package hugo - -import ( - "github.com/gardener/docforge/pkg/processors" - "github.com/gardener/docforge/pkg/writers" -) - -// Options is the configuration options for creating Hugo implementations -// docforge interfaces -type Options struct { - // PrettyUrls indicates if links will rewritten for Hugo will be - // formatted for pretty url support or not. Pretty urls in Hugo - // place built source content in index.html, which resides in a path segment with - // the name of the file, making request URLs more resource-oriented. - // Example: (source) sample.md -> (build) sample/index.html -> (runtime) ./sample - PrettyUrls bool - // IndexFileNames defines a list of file names that indicate - // their content can be used as Hugo section files (_index.md). - IndexFileNames []string - // Writer is the underlying writer used by hugo#FSWriter to serialize - // content - Writer writers.Writer - // BaseURL is used from the Hugo processor to rewrite relative links to root-relative - BaseURL string -} - -// NewWriter creates a new Hugo Writer implementing writers#Writer -func NewWriter(opts *Options) writers.Writer { - return &FSWriter{ - opts.Writer, - } -} - -// NewProcessor creates a new Hugo WriProcess implementing processors#Processor -func NewProcessor(opts *Options) processors.Processor { - return &Processor{ - opts.PrettyUrls, - opts.IndexFileNames, - opts.BaseURL, - } -} diff --git a/pkg/hugo/processor.go b/pkg/hugo/processor.go deleted file mode 100755 index 141f7ca5..00000000 --- a/pkg/hugo/processor.go +++ /dev/null @@ -1,161 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package hugo - -import ( - "fmt" - "net/url" - "strings" - - "github.com/gardener/docforge/pkg/api" - "github.com/gardener/docforge/pkg/markdown" - "github.com/gardener/docforge/pkg/markdown/parser" - "github.com/gardener/docforge/pkg/processors" - "k8s.io/klog/v2" -) - -// Processor is a processor implementation responsible to rewrite links -// on document that use source format (/.md) to destination format -// (/ for sites configured for pretty URLs and /.html -// for sites configured for ugly URLs) -type Processor struct { - // PrettyUrls indicates if links will rewritten for Hugo will be - // formatted for pretty url support or not. Pretty urls in Hugo - // place built source content in index.html, which resides in a path segment with - // the name of the file, making request URLs more resource-oriented. - // Example: (source) sample.md -> (build) sample/index.html -> (runtime) ./sample - PrettyUrls bool - // IndexFileNames defines a list of file names that indicate - // their content can be used as Hugo section files (_index.md). - IndexFileNames []string - // BaseURL is the root relative path configured for the Hugo website - BaseURL string -} - -// Process implements Processor#Process -func (f *Processor) Process(document *processors.Document) error { - isNodeIndexFile := f.nodeIsIndexFile(document.Node.Name) - var ( - contentBytes []byte - err error - ) - p := parser.NewParser() - parsedDocument := p.Parse(document.DocumentBytes) - if contentBytes, err = markdown.UpdateMarkdownLinks(parsedDocument, func(markdownType markdown.Type, destination, text, title []byte) ([]byte, []byte, []byte, error) { - // quick sanity check for ill-parsed links if any - if destination == nil { - return destination, text, title, nil - } - link := document.GetLinkByDestination(string(destination)) - if link == nil { - return destination, text, title, nil - } - return f.rewriteDestination(destination, text, title, document.Node.Name, isNodeIndexFile, link) - }); err != nil { - return err - } - if contentBytes, err = markdown.UpdateHTMLLinksRefs(contentBytes, func(isImage bool, url []byte) ([]byte, error) { - link := document.GetLinkByDestination(string(url)) - if link == nil { - return url, nil - } - destination, _, _, err := f.rewriteDestination([]byte(url), []byte(""), []byte(""), document.Node.Name, isNodeIndexFile, link) - if err != nil { - return url, err - } - return destination, err - }); err != nil { - return err - } - document.DocumentBytes = contentBytes - return nil -} - -func (f *Processor) rewriteDestination(destination, text, title []byte, nodeName string, isNodeIndexFile bool, l *processors.Link) ([]byte, []byte, []byte, error) { - if len(destination) == 0 { - return destination, nil, nil, nil - } - link := string(destination) - link = strings.TrimSpace(link) - // trim leading and trailing quotes if any - link = strings.TrimSuffix(strings.TrimPrefix(link, "\""), "\"") - u, err := url.Parse(link) - if err != nil { - klog.Warning("Invalid link:", link) - return destination, text, title, nil - } - if !u.IsAbs() && !strings.HasPrefix(link, "/") && !strings.HasPrefix(link, "#") { - _l := link - link = u.Path - if l.DestinationNode != nil { - absPath := api.Path(l.DestinationNode, "/") - link = fmt.Sprintf("%s/%s/%s", f.BaseURL, absPath, strings.ToLower(l.DestinationNode.Name)) - } - if l.IsResource { - for strings.HasPrefix(link, "../") { - link = strings.TrimPrefix(link, "../") - } - link = fmt.Sprintf("%s/%s", f.BaseURL, link) - } - - link = strings.TrimPrefix(link, "./") - if f.PrettyUrls { - link = strings.TrimSuffix(link, ".md") - // Remove the last path segment if it is readme, index or _index - // The Hugo writer will rename those files to _index.md and runtime - // references will be to the sections in which they reside. - for _, s := range f.IndexFileNames { - if strings.HasSuffix(strings.ToLower(link), s) { - pathSegments := strings.Split(link, "/") - if len(pathSegments) > 0 { - pathSegments = pathSegments[:len(pathSegments)-1] - link = strings.Join(pathSegments, "/") - } - break - } - } - } else { - if strings.HasSuffix(link, ".md") { - link = strings.TrimSuffix(link, ".md") - // TODO: propagate fragment and query if any - link = fmt.Sprintf("%s.html", link) - } - } - if l.DestinationNode != nil { - // check for hugo url property & rewrite the link - // see https://gohugo.io/content-management/urls/ for details - if val, ok := l.DestinationNode.Properties["frontmatter"]; ok { - if fmProps, ok := val.(map[string]interface{}); ok { - if urlVal, ok := fmProps["url"]; ok { - if urlStr, ok := urlVal.(string); ok { - if _, err = url.Parse(urlStr); err != nil { - klog.Warningf("Invalid frontmatter url: %s for %s\n", urlStr, l.DestinationNode.Source) - } else { - link = urlStr - } - } - } - } - } - } - if _l != link { - klog.V(6).Infof("[%s] Rewriting node link for Hugo: %s -> %s \n", nodeName, _l, link) - } - return []byte(link), text, title, nil - } - return destination, text, title, nil -} - -// Compares a node name to the configured list of index file -// and a default name '_index.md' to determine if this node -// is an index document node. -func (f *Processor) nodeIsIndexFile(name string) bool { - for _, s := range f.IndexFileNames { - if strings.EqualFold(name, s) { - return true - } - } - return name == "_index.md" -} diff --git a/pkg/hugo/processor_test.go b/pkg/hugo/processor_test.go deleted file mode 100755 index da1a1dc5..00000000 --- a/pkg/hugo/processor_test.go +++ /dev/null @@ -1,295 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package hugo - -import ( - "bytes" - "testing" - - "github.com/gardener/docforge/pkg/api" - "github.com/gardener/docforge/pkg/processors" - "github.com/gardener/docforge/pkg/util/tests" - "github.com/stretchr/testify/assert" - "k8s.io/utils/pointer" -) - -func init() { - tests.SetKlogV(6) -} - -func TestHugoProcess(t *testing.T) { - testCases := []struct { - name string - in []byte - node *api.Node - want []byte - wantErr error - mutate func(p *Processor, node *api.Node) - links []*processors.Link - }{ - { - name: "", - in: []byte(`[GitHub](./a/b.md)`), - node: &api.Node{Name: "Test"}, - want: []byte("[GitHub](/a/b)"), - mutate: func(p *Processor, node *api.Node) { - p.PrettyUrls = true - }, - links: []*processors.Link{ - { - DestinationNode: createNodeWithParents("b.md", "a"), - Destination: pointer.StringPtr("./a/b.md"), - }, - }, - }, - { - name: "", - in: []byte(`B`), - node: &api.Node{Name: "Test"}, - want: []byte(`B`), - mutate: nil, - }, - { - name: "", - in: []byte(`B`), - node: &api.Node{Name: "Test"}, - want: []byte(`B`), - mutate: func(p *Processor, node *api.Node) { - p.PrettyUrls = true - }, - links: []*processors.Link{ - { - DestinationNode: createNodeWithParents("b.md", "a"), - Destination: pointer.StringPtr("a/b.md"), - }, - }, - }, - } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - p := &Processor{} - if tc.mutate != nil { - tc.mutate(p, tc.node) - } - document := &processors.Document{ - Node: tc.node, - DocumentBytes: tc.in, - Links: tc.links, - } - err := p.Process(document) - - if tc.wantErr != err { - t.Errorf("want err %v != %v", tc.wantErr, err) - } - assert.Equal(t, string(tc.want), string(document.DocumentBytes)) - }) - } -} - -func TestRewriteDestination(t *testing.T) { - testCases := []struct { - name string - destination string - text string - title string - nodeName string - isNodeIndexFile bool - wantDestination string - wantText string - wantTitle string - wantError error - mutate func(h *Processor) - destinationNode *api.Node - isResource bool - }{ - { - name: "doesn't_change_links_that_begin_with_#", - destination: "#fragment-id", - text: "", - title: "", - nodeName: "testnode", - isNodeIndexFile: false, - wantDestination: "#fragment-id", - wantTitle: "", - wantText: "", - wantError: nil, - mutate: nil, - destinationNode: nil, - }, - { - name: "doesn't_change_absolute_links", - destination: "https://github.com/a/b/sample.md", - text: "", - title: "", - nodeName: "testnode", - isNodeIndexFile: false, - wantDestination: "https://github.com/a/b/sample.md", - wantTitle: "", - wantText: "", - wantError: nil, - mutate: nil, - destinationNode: nil, - }, - { - name: "to_relative_link_when_node_is_part_of_structure", - destination: "./a/b/sample.md", - text: "", - title: "", - nodeName: "testnode", - isNodeIndexFile: false, - wantDestination: "/a/b/sample", - wantTitle: "", - wantText: "", - wantError: nil, - mutate: func(h *Processor) { h.PrettyUrls = true }, - destinationNode: createNodeWithParents("sample.md", "b", "a"), - }, - { - name: "changes_file_extension_when_pretty_urls_is_disabled", - destination: "./a/b/README.md", - text: "", - title: "", - nodeName: "testnode", - isNodeIndexFile: false, - wantDestination: "/a/b/readme.html", - wantTitle: "", - wantText: "", - wantError: nil, - mutate: nil, - destinationNode: createNodeWithParents("README.md", "b", "a"), - }, - { - name: "to_link_known_as_hugo_section_when_link_destination_matches_index_file_names", - destination: "./a/b/README.md", - text: "", - title: "", - nodeName: "testnode", - isNodeIndexFile: false, - wantDestination: "/base/url/a/b", - wantTitle: "", - wantText: "", - wantError: nil, - mutate: func(h *Processor) { - h.PrettyUrls = true - h.IndexFileNames = []string{"readme", "read.me", "index", "_index"} - h.BaseURL = "/base/url" - }, - destinationNode: createNodeWithParents("README.md", "b", "a"), - }, - { - name: "to_the_root_relative_link_of_a_resource", - destination: "images/1.png", - isNodeIndexFile: true, - nodeName: "_index.md", - wantDestination: "/images/1.png", - mutate: func(h *Processor) { - h.PrettyUrls = true - }, - isResource: true, - }, - { - name: "to_the_root_relative_link_of_a_resource", - destination: "../../images/1.png", - isNodeIndexFile: true, - nodeName: "_index.md", - wantDestination: "/images/1.png", - mutate: func(h *Processor) { - h.PrettyUrls = true - }, - isResource: true, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - p := &Processor{} - if tc.mutate != nil { - tc.mutate(p) - } - l := &processors.Link{ - DestinationNode: tc.destinationNode, - Destination: &tc.destination, - IsResource: tc.isResource, - } - gotDestination, gotText, gotTitle, gotErr := p.rewriteDestination([]byte(tc.destination), []byte(tc.text), []byte(tc.title), tc.nodeName, tc.isNodeIndexFile, l) - - if gotErr != tc.wantError { - t.Errorf("want error %v != %v", tc.wantError, gotErr) - } - if !bytes.Equal(gotDestination, []byte(tc.wantDestination)) { - t.Errorf("want destination %v != %v", tc.wantDestination, string(gotDestination)) - } - if !bytes.Equal(gotText, []byte(tc.wantText)) { - t.Errorf("want text %v != %v", tc.wantText, string(gotText)) - } - if !bytes.Equal(gotTitle, []byte(tc.wantTitle)) { - t.Errorf("want title %v != %v", tc.wantTitle, string(gotTitle)) - } - }) - } -} - -func createNodeWithParents(name string, parentNames ...string) *api.Node { - destinationNode := &api.Node{ - Name: name, - } - childNode := destinationNode - for _, n := range parentNames { - parent := &api.Node{ - Name: n, - } - childNode.SetParent(parent) - childNode = parent - } - - return destinationNode -} - -func TestProcessor_nodeIsIndexFile(t *testing.T) { - tests := []struct { - name string - IndexFileNames []string - nodeName string - want bool - }{ - { - name: "returns_true_when_compared_to_default_index_name", - IndexFileNames: []string{}, - nodeName: "_index.md", - want: true, - }, - { - name: "returns_false_if_not_equal_to_default_index_name", - IndexFileNames: []string{}, - nodeName: "someNodeName.md", - want: false, - }, - { - name: "returns_false_if_not_equal_to_default_index_name", - IndexFileNames: []string{ - "someNodeName.md", - }, - nodeName: "someNodeName.md", - want: true, - }, - { - name: "returns_true_ignoring_text_case_type", - IndexFileNames: []string{ - "someNodeName.md", - }, - nodeName: "somenodename.md", - want: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - f := &Processor{ - IndexFileNames: tt.IndexFileNames, - } - if got := f.nodeIsIndexFile(tt.nodeName); got != tt.want { - t.Errorf("Processor.nodeIsIndexFile() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/markdown/frontmatter.go b/pkg/markdown/frontmatter.go deleted file mode 100644 index f77e39cc..00000000 --- a/pkg/markdown/frontmatter.go +++ /dev/null @@ -1,199 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package markdown - -import ( - "bytes" - "errors" - "fmt" - "io" - "io/ioutil" - "reflect" - "strings" - - "gopkg.in/yaml.v3" -) - -// ErrFrontMatterNotClosed is raised to signal -// that the rules for defining a frontmatter element -// in a markdown document have been violated -var ErrFrontMatterNotClosed error = errors.New("Missing closing frontmatter `---` found") - -// StripFrontMatter splits a provided document into front-matter -// and content. -func StripFrontMatter(b []byte) ([]byte, []byte, error) { - var ( - started bool - yamlBeg int - yamlEnd int - contentStart int - ) - - buf := bytes.NewBuffer(b) - - for { - line, err := buf.ReadString('\n') - - if errors.Is(err, io.EOF) { - // handle documents that contain only forntmatter - // and no line ending after closing --- - if started && yamlEnd == 0 { - if l := strings.TrimSpace(line); l == "---" { - yamlEnd = len(b) - buf.Len() - len([]byte(line)) - contentStart = len(b) - } - - } - break - } - - if err != nil { - return nil, nil, err - } - - if l := strings.TrimSpace(line); l != "---" { - // Only whitespace is acceptable before front-matter - // Any other preceding text is interpeted as frontmater-less - // document - if !started && len(l) > 0 { - return nil, b, nil - } - continue - } - - if !started { - started = true - yamlBeg = len(b) - buf.Len() - } else { - yamlEnd = len(b) - buf.Len() - len([]byte(line)) - contentStart = yamlEnd + len([]byte(line)) - break - } - } - - if started && yamlEnd == 0 { - return nil, nil, ErrFrontMatterNotClosed - } - - fm := b[yamlBeg:yamlEnd] - content := b[contentStart:] - - return fm, content, nil -} - -// InsertFrontMatter prepends the content bytes with -// front matter enclosed in the standard marks --- -func InsertFrontMatter(fm []byte, content []byte) ([]byte, error) { - var ( - data []byte - err error - ) - if len(fm) < 1 { - return content, nil - } - buf := bytes.NewBuffer([]byte("---\n")) - buf.Write(fm) - // TODO: configurable empty line after frontmatter - buf.WriteString("---\n") - buf.Write(content) - if data, err = ioutil.ReadAll(buf); err != nil { - return nil, err - } - return data, nil -} - -// MatchFrontMatterRules inspects content `b` for compliance with `frontMatter` -// and `excludeFrontMatter`. -func MatchFrontMatterRules(b []byte, frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}) (bool, error) { - fmBytes, _, err := StripFrontMatter(b) - if err != nil { - return false, err - } - if fmBytes == nil { - return false, nil - } - fm := map[string]interface{}{} - if err := yaml.Unmarshal(fmBytes, fm); err != nil { - return false, err - } - selected := false - for path, v := range frontMatter { - if MatchFrontMatterRule(path, v, fm) { - selected = true - break - } - } - for path, v := range excludeFrontMatter { - if MatchFrontMatterRule(path, v, fm) { - selected = false - break - } - } - return selected, nil -} - -// MatchFrontMatterRule explores a parsed frontmatter object `data` to matchFMRule -// `value` at `path` pattern and return true on successfull matchFMRule or false -// otherwise. -// Path is an expression with a JSONPath-like simplified notation. -// An object in path is modeled as dot (`.`). Paths start with the root object, -// i.e. the most minimal path is `.`. -// An object element value is referenced by its name (key) in the object map: -// `.a.b.c` is path to element `c` in map `b` in map `a` in root object map. -// Element values can be scalar, object maps or arrays. -// An element in an array is referenced by its index: `.a.b[1]` references `b` -// array element with index 1. -// Paths can include up to one wildcard `**` symbol that models *any* path node. -// A `.a.**.c` models any path starting with `.a.` and ending with `.c`. -func MatchFrontMatterRule(path string, val interface{}, data interface{}) bool { - return matchFMRule(path, val, nil, data) -} - -func matchFMRule(pathPattern string, val interface{}, path []string, data interface{}) bool { - if path == nil { - path = []string{"."} - } - p := strings.Join(path, "") - if _matchFMPath(pathPattern, p) { - if reflect.DeepEqual(val, data) { - return true - } - } - switch vv := data.(type) { - case []interface{}: - for i, u := range vv { - _p := append(path, fmt.Sprintf("[%d]", i)) - if ok := matchFMRule(pathPattern, val, _p, u); ok { - return true - } - } - case map[string]interface{}: - for k, u := range vv { - if path[(len(path))-1] != "." { - path = append(path, ".") - } - _p := append(path, k) - if ok := matchFMRule(pathPattern, val, _p, u); ok { - return true - } - } - } - return false -} - -func _matchFMPath(pathPattern, path string) bool { - if pathPattern == path { - return true - } - s := strings.Split(pathPattern, "**") - if len(s) > 1 { - if strings.HasPrefix(path, s[0]) { - if len(s[1]) == 0 || strings.HasSuffix(path, s[1]) { - return true - } - } - } - return false -} diff --git a/pkg/markdown/frontmatter_test.go b/pkg/markdown/frontmatter_test.go deleted file mode 100644 index 71783608..00000000 --- a/pkg/markdown/frontmatter_test.go +++ /dev/null @@ -1,223 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package markdown - -import ( - "testing" -) - -func TestStripFrontMatter(t *testing.T) { - testCases := []struct { - in string - wantFM string - wantContent string - wantErr error - }{ - { - in: `--- -title: Test -prop: A ---- - -# Head 1 -`, - wantFM: `title: Test -prop: A -`, - wantContent: ` -# Head 1 -`, - }, - { - in: `# Head 1`, - wantFM: "", - wantContent: `# Head 1`, - }, - { - in: `--- -Title: A -`, - wantFM: "", - wantContent: "", - wantErr: ErrFrontMatterNotClosed, - }, - { - in: `Some text - ---- -`, - wantFM: "", - wantContent: `Some text - ---- -`, - }, - { - in: `--- -title: Core Components ----`, - wantFM: "title: Core Components\n", - wantContent: "", - }, - } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - fm, c, err := StripFrontMatter([]byte(tc.in)) - if err != tc.wantErr { - t.Errorf("expected err %v != %v", tc.wantErr, err) - } - if string(fm) != tc.wantFM { - t.Errorf("\nwant frontmatter:\n%s\ngot:\n%s\n", tc.wantFM, fm) - } - if string(c) != tc.wantContent { - t.Errorf("\nwant content:\n%s\ngot:\n%s\n", tc.wantContent, c) - } - }) - } -} - -func TestMatchFrontMatterRule(t *testing.T) { - testCases := []struct { - path string - value interface{} - data map[string]interface{} - wantMatch bool - }{ - { - path: ".A.B", - value: 5, - data: map[string]interface{}{ - "A": map[string]interface{}{ - "B": 5, - }, - }, - wantMatch: true, - }, - { - path: ".A.B", - value: true, - data: map[string]interface{}{ - "A": map[string]interface{}{ - "B": true, - }, - }, - wantMatch: true, - }, - { - path: ".A.B", - value: "a", - data: map[string]interface{}{ - "A": map[string]interface{}{ - "B": "a", - }, - }, - wantMatch: true, - }, - { - path: ".A.**.C", - value: 5, - data: map[string]interface{}{ - "A": map[string]interface{}{ - "B": map[string]interface{}{ - "C": 5, - }, - }, - }, - wantMatch: true, - }, - { - path: ".**", - value: 5, - data: map[string]interface{}{ - "A": map[string]interface{}{ - "B": map[string]interface{}{ - "C": 5, - }, - }, - }, - wantMatch: true, - }, - { - path: ".A.B[1]", - value: "b", - data: map[string]interface{}{ - "A": map[string]interface{}{ - "B": []interface{}{"a", "b", "c"}, - }, - }, - wantMatch: true, - }, - { - path: ".A.B[1].C2", - value: 2, - data: map[string]interface{}{ - "A": map[string]interface{}{ - "B": []interface{}{ - map[string]interface{}{ - "C1": 1, - }, - map[string]interface{}{ - "C2": 2, - }, - map[string]interface{}{ - "C3": 3, - }, - }, - }, - }, - wantMatch: true, - }, - { - path: ".A.**.C2", - value: 2, - data: map[string]interface{}{ - "A": map[string]interface{}{ - "B": []interface{}{ - map[string]interface{}{ - "C1": 1, - }, - map[string]interface{}{ - "C2": 2, - }, - map[string]interface{}{ - "C3": 3, - }, - }, - }, - }, - wantMatch: true, - }, - { - path: ".A.B", - value: 2, - data: map[string]interface{}{ - "A": map[string]interface{}{ - "B": 5, - }, - }, - wantMatch: false, - }, - { - path: ".A", - value: map[string]interface{}{ - "B": []interface{}{"a", "b", "c"}, - }, - data: map[string]interface{}{ - "A": map[string]interface{}{ - "B": []interface{}{"a", "b", "c"}, - }, - }, - wantMatch: true, - }, - } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - matched := MatchFrontMatterRule(tc.path, tc.value, tc.data) - if tc.wantMatch && !matched { - t.Errorf("expected a match for path %s, got no match", tc.path) - } - }) - } -} diff --git a/pkg/markdown/html_links.go b/pkg/markdown/html_links.go deleted file mode 100644 index f7e1483f..00000000 --- a/pkg/markdown/html_links.go +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package markdown - -import ( - "fmt" - "regexp" - "strings" - - "github.com/hashicorp/go-multierror" -) - -var ( - htmlLinkTagPattern = regexp.MustCompile(`(?P<\b[^>]*?\b((?i)href|(?i)src)\s*=\s*)((\"(?P[^"]*)\")|('(?P[^']*)')|(?P[^'">\s]+))`) -) - -// UpdateHTMLLinkRef is a callback function invoked by UpdateHTMLLinksRefs on -// each link (src|href attribute) found in an HTML tag. -// It is supplied the isImage flag and the link reference destination and is expected to return -// a destination that will be used to update the link , if different, or error. -type UpdateHTMLLinkRef func(isImage bool, destination []byte) ([]byte, error) - -// UpdateHTMLLinksRefs matches links in HTML tags in a document content and -// invokes the supplied updateRef callback function supplying the link -// reference as argument and using the function call result to update the -// destination of the matched link. -func UpdateHTMLLinksRefs(documentBytes []byte, updateRef UpdateHTMLLinkRef) ([]byte, error) { - if updateRef == nil { - return documentBytes, nil - } - var errors *multierror.Error - documentBytes = htmlLinkTagPattern.ReplaceAllFunc(documentBytes, func(match []byte) []byte { - matchedLink := string(match) - url := extractLink(matchedLink) - if len(url) == 0 { - return match - } - var prefix, suffix string - prefixSufix := strings.Split(matchedLink, url) - prefix = prefixSufix[0] - isImage := strings.HasPrefix(prefix, " 1 { - suffix = prefixSufix[1] - } - destination, err := updateRef(isImage, []byte(url)) - if err != nil { - errors = multierror.Append(errors, err) - return match - } - newLinkTag := fmt.Sprintf("%s%s%s", prefix, destination, suffix) - return []byte(newLinkTag) - }) - return documentBytes, errors.ErrorOrNil() -} - -func extractLink(matchedLink string) string { - subMatch := htmlLinkTagPattern.FindStringSubmatch(matchedLink) - var linkIndecies []int - for i, name := range htmlLinkTagPattern.SubexpNames() { - if name == "Link" { - linkIndecies = append(linkIndecies, i) - } - } - - for _, linkIndex := range linkIndecies { - if len(subMatch[linkIndex]) != 0 { - return subMatch[linkIndex] - } - } - - return "" -} diff --git a/pkg/markdown/link_modifier.go b/pkg/markdown/link_modifier.go new file mode 100644 index 00000000..ae472127 --- /dev/null +++ b/pkg/markdown/link_modifier.go @@ -0,0 +1,987 @@ +// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package markdown + +import ( + "bufio" + "bytes" + "fmt" + "github.com/yuin/goldmark/ast" + extast "github.com/yuin/goldmark/extension/ast" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" + "golang.org/x/net/html" + "gopkg.in/yaml.v3" + "io" + "regexp" + "sync" +) + +var ( + // pool with reusable buffers + bufPool = sync.Pool{ + New: func() interface{} { + return new(bytes.Buffer) + }, + } + // defines an ordered list item marker without next not space char '[^ ]+' + marker = regexp.MustCompile(`^(\d{1,9}[.)]) {1,4}`) + // defines a fence block line + fence = regexp.MustCompile("^ {0,3}```") + // GFM autolink extensions + http = regexp.MustCompile(`^https*://([a-zA-Z\d\-_]+\.)*[a-zA-Z\d\-]+\.[a-zA-Z\d\-]+[^ <]*$`) + www = regexp.MustCompile(`^www\.([a-zA-Z\d\-_]+\.)*[a-zA-Z\d\-]+\.[a-zA-Z\d\-]+[^ <]*$`) + email = regexp.MustCompile(`^[a-zA-Z\d.\-+]+@([a-zA-Z\d\-_]+\.)+[a-zA-Z\d\-_]+$`) +) + +// ResolveLink type defines function for modifying link destination +// dest - original destination +// isEmbeddable - if true, raw destination required +type ResolveLink func(dest string, isEmbeddable bool) (string, error) + +// resolveSame implements markdown.ResolveLink - the result is the same as input +// used if WithLinkResolver option is not set +func resolveSame(dest string, _ bool) (string, error) { + return dest, nil +} + +// LinkResolver is an option name used in WithLinkResolver. +const optLinkResolver renderer.OptionName = "LinkResolver" + +type withLinkResolver struct { + value ResolveLink +} + +func (o *withLinkResolver) SetConfig(c *renderer.Config) { + c.Options[optLinkResolver] = o.value +} + +// WithLinkResolver is a functional option that allow you to set the ResolveLink to the renderer. +func WithLinkResolver(linkResolver ResolveLink) renderer.Option { + return &withLinkResolver{linkResolver} +} + +// A linkModifierRenderer struct is an implementation of renderer.Renderer interface. +type linkModifierRenderer struct { + config *renderer.Config +} + +// NewLinkModifierRenderer returns a new linkModifierRenderer with given renderer options. +func NewLinkModifierRenderer(opts ...renderer.Option) renderer.Renderer { + config := renderer.NewConfig() + for _, opt := range opts { + opt.SetConfig(config) + } + if _, ok := config.Options[optLinkResolver]; !ok { + WithLinkResolver(resolveSame).SetConfig(config) + } + return &linkModifierRenderer{ + config: config, + } +} + +func (l *linkModifierRenderer) AddOptions(opts ...renderer.Option) { + for _, opt := range opts { + opt.SetConfig(l.config) + } +} + +func (l *linkModifierRenderer) Render(w io.Writer, source []byte, node ast.Node) error { + // walk & render nodes + r := &Renderer{ + source: source, + linkResolver: l.config.Options[optLinkResolver].(ResolveLink), + indents: make([]byte, 0, 20), + markers: make([]int, 0, 5), + emphasis: make([]byte, 0, 5), + } + writer, ok := w.(*bytes.Buffer) + if ok { + r.writer = writer + } else { + r.writer = &bytes.Buffer{} + } + err := ast.Walk(node, func(node ast.Node, entering bool) (ast.WalkStatus, error) { + switch node.Kind() { + case ast.KindDocument: + return r.renderDocument(node, entering) + // commonmark container blocks + case ast.KindBlockquote: + return r.renderBlockquote(node, entering) + case ast.KindList: + return r.renderList(node, entering) + case ast.KindListItem: + return r.renderListItem(node, entering) + // commonmark blocks + case ast.KindHeading: + return r.renderHeading(node, entering) + case ast.KindCodeBlock, ast.KindFencedCodeBlock: + return r.renderFencedCodeBlock(node, entering) + case ast.KindHTMLBlock: + return r.renderHTMLBlock(node, entering) + case ast.KindParagraph: + return r.renderParagraph(node, entering) + case ast.KindTextBlock: + return r.renderTextBlock(node, entering) + case ast.KindThematicBreak: + return r.renderThematicBreak(node, entering) + // commonmark inlines + case ast.KindAutoLink: + return r.renderAutoLink(node, entering) + case ast.KindCodeSpan: + return r.renderCodeSpan(node, entering) + case ast.KindEmphasis: + return r.renderEmphasis(node, entering) + case ast.KindLink: + return r.renderLink(node, entering) + case ast.KindImage: + return r.renderImage(node, entering) + case ast.KindRawHTML: + return r.renderRawHTML(node, entering) + case ast.KindText, ast.KindString: + return r.renderText(node, entering) + // GFM extension blocks + case extast.KindTable: + return r.renderTable(node, entering) + case extast.KindTableHeader: + return r.renderTableHeader(node, entering) + case extast.KindTableRow: + return r.renderTableRow(node, entering) + case extast.KindTableCell: + return r.renderTableCell(node, entering) + // GFM extension inlines + case extast.KindTaskCheckBox: + return r.renderTaskCheckBox(node, entering) + case extast.KindStrikethrough: + return r.renderStrikethrough(node, entering) + default: + return ast.WalkContinue, nil + } + }) + if !ok { + _, _ = w.Write(r.writer.Bytes()) + } + return err +} + +// Renderer holds document source, buffer writer, info for indents and some nodes for rendering a markdown +type Renderer struct { + source []byte + writer *bytes.Buffer + linkResolver ResolveLink + indents []byte + markers []int + emphasis []byte + table bool +} + +// --------------------------- Node Renders + +func (r *Renderer) renderDocument(node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.Document) + if entering { + var err error + // process frontmatter + fm := n.Meta() + if len(fm) > 0 { + _, _ = r.writer.Write([]byte("---\n")) + var cnt []byte + cnt, err = yaml.Marshal(fm) + if err != nil { + return ast.WalkStop, err + } + _, _ = r.writer.Write(cnt) + _, _ = r.writer.Write([]byte("---\n")) + if n.HasChildren() { + r.newLine(false) + } + } + } else { + if n.HasChildren() { + cnt := r.writer.Bytes() + if len(cnt) > 0 && cnt[len(cnt)-1] != '\n' { + r.newLine(false) + } + } + } + return ast.WalkContinue, nil +} + +// commonmark container blocks + +func (r *Renderer) renderBlockquote(n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + r.blockSeparator(n) + // no laziness - block new lines will always start with '>' + _, _ = r.writer.Write([]byte("> ")) + r.indents = append(r.indents, '>', ' ') + } else { + r.indents = r.indents[:len(r.indents)-2] + breakBlockquoteLazyContinuation(n.NextSibling()) + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderList(n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + r.blockSeparator(n) + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderListItem(node ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + n := node.(*ast.ListItem) + r.blockSeparator(n) + listMarker := buildListMarker(n) + _, _ = r.writer.Write(listMarker) + r.markers = append(r.markers, len(listMarker)) + r.indents = append(r.indents, bytes.Repeat([]byte{' '}, len(listMarker))...) + } else { + r.indents = r.indents[:len(r.indents)-r.markers[len(r.markers)-1]] + r.markers = r.markers[:len(r.markers)-1] + } + return ast.WalkContinue, nil +} + +// commonmark blocks + +func (r *Renderer) renderHeading(node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.Heading) + atx := true // defaults to ATX headings + if n.Lines().Len() > 1 && n.Level <= 2 { + atx = false // multiline heading -> use Setext headings + } + if entering { + r.blockSeparator(n) + if atx { + _, _ = r.writer.Write(bytes.Repeat([]byte{'#'}, n.Level)) + _ = r.writer.WriteByte(' ') + } + } else { + if !atx { + r.newLine(true) + if n.Level == 1 { + _, _ = r.writer.Write([]byte{'=', '=', '='}) + } else { + _, _ = r.writer.Write([]byte{'-', '-', '-'}) + } + } + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderFencedCodeBlock(n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + buf := bufPool.Get().(*bytes.Buffer) + defer bufPool.Put(buf) + buf.Reset() + indents := len(r.indents) > 0 + var fb byte = '`' + segments := n.Lines() + for _, l := range segments.Sliced(0, segments.Len()) { + if fence.Match(l.Value(r.source)) { + fb = '~' + } + if indents { + _, _ = buf.Write(r.indents) + } + _, _ = buf.Write(l.Value(r.source)) + } + r.blockSeparator(n) + _, _ = r.writer.Write([]byte{fb, fb, fb}) + if n.Kind() == ast.KindFencedCodeBlock { + fn := n.(*ast.FencedCodeBlock) + language := fn.Language(r.source) + if language != nil { + _, _ = r.writer.Write(language) + } + } + r.newLine(false) + if buf.Len() > 0 { + _, _ = r.writer.Write(buf.Bytes()) + } + if indents { + _, _ = r.writer.Write(r.indents) + } + _, _ = r.writer.Write([]byte{fb, fb, fb}) + } + return ast.WalkSkipChildren, nil +} + +func (r *Renderer) renderHTMLBlock(node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.HTMLBlock) + if entering { + r.blockSeparator(n) + // HTMLBlockType 6 & 7 may contain links and images + if n.HTMLBlockType >= ast.HTMLBlockType6 { + buf := bufPool.Get().(*bytes.Buffer) + defer bufPool.Put(buf) + buf.Reset() + r.writeSegments(buf, n.Lines(), false) + // modify + modBuf := bufPool.Get().(*bytes.Buffer) + defer bufPool.Put(modBuf) + modBuf.Reset() + modified, err := r.modifyHTMLTags(buf.Bytes(), modBuf) + if err != nil { + return ast.WalkStop, err + } + if modified { + buf = modBuf + } + r.writeContent(buf.Bytes()) + } else { + r.writeSegments(r.writer, n.Lines(), len(r.indents) > 0) + // HTMLBlockType 1 to 5 end condition is not blank line + if n.HasClosure() { + // line that contains end condition for blocks with type < 6 + if len(r.indents) > 0 { + _, _ = r.writer.Write(r.indents) + } + _, _ = r.writer.Write(n.ClosureLine.Value(r.source)) + } + } + } + return ast.WalkSkipChildren, nil +} + +func (r *Renderer) renderParagraph(n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + r.blockSeparator(n) + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderTextBlock(n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + r.blockSeparator(n) + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderThematicBreak(node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.ThematicBreak) + if entering { + r.blockSeparator(n) + if n.HasBlankPreviousLines() { + _, _ = r.writer.Write([]byte{'-', '-', '-'}) + } else { + // as '-' could be Setext heading 2 use '*' + _, _ = r.writer.Write([]byte{'*', '*', '*'}) + } + } + return ast.WalkSkipChildren, nil +} + +// commonmark inlines + +func (r *Renderer) renderAutoLink(node ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + n := node.(*ast.AutoLink) + label := n.Label(r.source) + if bytes.HasPrefix(bytes.ToLower(label), []byte("mailto:")) { + n.AutoLinkType = ast.AutoLinkEmail // fix the node type + } + classic := isClassicAutolink(label, r.writer.Bytes()) + if classic { + _ = r.writer.WriteByte('<') + } + if n.AutoLinkType == ast.AutoLinkURL { + dest, err := r.linkResolver(string(label), false) + if err != nil { + return ast.WalkStop, err + } + _, _ = r.writer.Write([]byte(dest)) + } else { + _, _ = r.writer.Write(label) + } + if classic { + _ = r.writer.WriteByte('>') + } + } + return ast.WalkSkipChildren, nil +} + +func (r *Renderer) renderCodeSpan(n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + cs := []byte{'`'} + txt := n.Text(r.source) + c := bytes.Count(txt, []byte{'`'}) // if odd backtick count use "``" as code span string + if c%2 == 0 { + if n.PreviousSibling() != nil { + idx := bytes.LastIndexByte(r.writer.Bytes(), '\n') + if idx == -1 { + idx = 0 + } + c = bytes.Count(r.writer.Bytes()[idx:], []byte{'`'}) // if previous text has odd backtick count use "``" + } + } + if c%2 != 0 { + cs = append(cs, '`') + } + // if text starts or ends with '`' or ' ' add space + space := len(txt) > 0 && (txt[0] == '`' || txt[0] == ' ' || txt[len(txt)-1] == '`' || txt[len(txt)-1] == ' ') + if space && len(txt) == bytes.Count(txt, []byte{' '}) { + // if txt contains only spaces don't add additional space + space = false + } + _, _ = r.writer.Write(cs) + if space { + _ = r.writer.WriteByte(' ') + } + if r.table { // parser unescape '|' in code span + txt = escapePipes(txt) + } + txt = bytes.ReplaceAll(txt, []byte{'\n'}, []byte{' '}) // replace new lines with spaces + _, _ = r.writer.Write(txt) + if space { + _ = r.writer.WriteByte(' ') + } + _, _ = r.writer.Write(cs) + } + return ast.WalkSkipChildren, nil +} + +func (r *Renderer) renderEmphasis(node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.Emphasis) + if entering { + ch, _ := r.calcEmphasisChar(n) + _, _ = r.writer.Write(bytes.Repeat([]byte{ch}, n.Level)) + r.emphasis = append(r.emphasis, ch) + } else { + _, _ = r.writer.Write(bytes.Repeat([]byte{r.emphasis[len(r.emphasis)-1]}, n.Level)) + r.emphasis = r.emphasis[:len(r.emphasis)-1] + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderLink(node ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + _ = r.writer.WriteByte('[') + } else { + n := node.(*ast.Link) + _ = r.writer.WriteByte(']') + _ = r.writer.WriteByte('(') + dest, err := r.linkResolver(string(n.Destination), false) + if err != nil { + return ast.WalkStop, err + } + wrap := wrapLinkDestination([]byte(dest)) + if wrap { + _ = r.writer.WriteByte('<') + } + _, _ = r.writer.Write([]byte(dest)) + if wrap { + _ = r.writer.WriteByte('>') + } + if n.Title != nil { + q := getLinkTitleWrapper(n.Title) + _ = r.writer.WriteByte(' ') + _ = r.writer.WriteByte(q) + r.writeContent(n.Title) // support multi-line title + if q == '(' { + q = ')' + } + _ = r.writer.WriteByte(q) + } + _ = r.writer.WriteByte(')') + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderImage(node ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + _ = r.writer.WriteByte('!') + _ = r.writer.WriteByte('[') + } else { + n := node.(*ast.Image) + _ = r.writer.WriteByte(']') + _ = r.writer.WriteByte('(') + dest, err := r.linkResolver(string(n.Destination), true) + if err != nil { + return ast.WalkStop, err + } + wrap := wrapLinkDestination([]byte(dest)) + if wrap { + _ = r.writer.WriteByte('<') + } + _, _ = r.writer.Write([]byte(dest)) + if wrap { + _ = r.writer.WriteByte('>') + } + if n.Title != nil { + q := getLinkTitleWrapper(n.Title) + _ = r.writer.WriteByte(' ') + _ = r.writer.WriteByte(q) + r.writeContent(n.Title) // support multi-line title + if q == '(' { + q = ')' + } + _ = r.writer.WriteByte(q) + } + _ = r.writer.WriteByte(')') + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderRawHTML(node ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + n := node.(*ast.RawHTML) + buf := bufPool.Get().(*bytes.Buffer) + defer bufPool.Put(buf) + buf.Reset() + r.writeSegments(buf, n.Segments, false) + // modify + modBuf := bufPool.Get().(*bytes.Buffer) + defer bufPool.Put(modBuf) + modBuf.Reset() + modified, err := r.modifyHTMLTags(buf.Bytes(), modBuf) + if err != nil { + return ast.WalkStop, err + } + if modified { + buf = modBuf + } + r.writeContent(buf.Bytes()) + } + return ast.WalkSkipChildren, nil +} + +func (r *Renderer) renderText(node ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + n := node.(*ast.Text) + txt := n.Text(r.source) + r.additionalIndents(txt, n) + if n.HardLineBreak() || n.SoftLineBreak() || nextIsLineBreak(node.NextSibling(), r.source) { + // trim trailing spaces + txt = bytes.TrimRight(txt, " ") + } + _, _ = r.writer.Write(txt) + indents := len(r.indents) > 0 + if n.HardLineBreak() { + _ = r.writer.WriteByte(' ') + _ = r.writer.WriteByte(' ') + r.newLine(indents) + } else if n.SoftLineBreak() { + r.newLine(indents) + } + } + return ast.WalkSkipChildren, nil +} + +// GFM extension blocks + +func (r *Renderer) renderTable(n ast.Node, entering bool) (ast.WalkStatus, error) { + // https://pkg.go.dev/text/tabwriter - for pretty table writing + if entering { + // 'blankPreviousLines' is not propagated during transformations, so previous blank line is set + n.SetBlankPreviousLines(true) + r.blockSeparator(n) + r.table = true + } else { + r.table = false + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderTableHeader(node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + n := node.(*extast.TableHeader) + if n.Alignments == nil { + if t, ok := n.Parent().(*extast.Table); ok { + n.Alignments = t.Alignments + } + } + // close headers cells + _ = r.writer.WriteByte('|') + r.newLine(len(r.indents) > 0) + // write alignments + for _, a := range n.Alignments { + _ = r.writer.WriteByte('|') + align := []byte(" --- ") + switch a { + case extast.AlignLeft: + align = []byte(" :-- ") + case extast.AlignRight: + align = []byte(" --: ") + case extast.AlignCenter: + align = []byte(" :-: ") + } + _, _ = r.writer.Write(align) + } + _ = r.writer.WriteByte('|') + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderTableRow(_ ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + r.newLine(len(r.indents) > 0) + } else { + _ = r.writer.WriteByte('|') + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderTableCell(_ ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + _ = r.writer.WriteByte('|') + _ = r.writer.WriteByte(' ') + } else { + _ = r.writer.WriteByte(' ') + } + return ast.WalkContinue, nil +} + +// GFM extension inlines + +func (r *Renderer) renderTaskCheckBox(node ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + n := node.(*extast.TaskCheckBox) + if n.IsChecked { + _, _ = r.writer.Write([]byte("[X] ")) + } else { + _, _ = r.writer.Write([]byte("[ ] ")) + } + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderStrikethrough(_ ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + _, _ = r.writer.Write([]byte("~~")) + } else { + _, _ = r.writer.Write([]byte("~~")) + } + return ast.WalkContinue, nil +} + +// --------------------------- + +func (r *Renderer) newLine(indents bool) { + _ = r.writer.WriteByte('\n') + if indents { + _, _ = r.writer.Write(r.indents) + } +} + +// separates blocks +func (r *Renderer) blockSeparator(n ast.Node) { + if n.PreviousSibling() != nil { + // add new line to start a blocks + // for some blocks like HTMLBlock content is raw and ends with '\n', so skip new line in such case + cnt := r.writer.Bytes() + if len(cnt) > 0 && cnt[len(cnt)-1] != '\n' { + _ = r.writer.WriteByte('\n') + } + // add indents after new line + if len(r.indents) > 0 { + _, _ = r.writer.Write(r.indents) + } + // add a blank line between blocks in case there are blank previous lines + // in blockquote scope 'blankPreviousLines' flag is not calculated properly (always `false`), so blank line + // should be added in some cases + if n.HasBlankPreviousLines() || r.blankLineInBlockquoteScope(n) { + _ = r.writer.WriteByte('\n') + if len(r.indents) > 0 { + _, _ = r.writer.Write(r.indents) + } + } + } +} + +// returns true if blank line should be added between Text inlines or Paragraph|TestBlock blocks in blockquote scope +func (r *Renderer) blankLineInBlockquoteScope(n ast.Node) bool { + if bytes.IndexByte(r.indents, '>') != -1 && n.PreviousSibling() != nil { + k := n.Kind() + if k == ast.KindText || k == ast.KindParagraph || k == ast.KindTextBlock { + pk := n.PreviousSibling().Kind() + return pk == ast.KindText || pk == ast.KindParagraph || pk == ast.KindTextBlock + } + } + return false +} + +func (r *Renderer) writeSegments(w io.Writer, segments *text.Segments, indents bool) { + for i, l := range segments.Sliced(0, segments.Len()) { + if indents && i > 0 { + _, _ = w.Write(r.indents) + } + _, _ = w.Write(l.Value(r.source)) + } +} + +func (r *Renderer) writeContent(b []byte) { + if len(r.indents) == 0 { + _, _ = r.writer.Write(b) + } else { + reader := bufio.NewReader(bytes.NewReader(b)) + for i := 0; ; i++ { + l, err := reader.ReadBytes('\n') + if len(l) > 0 { + if i > 0 { + _, _ = r.writer.Write(r.indents) + } + _, _ = r.writer.Write(l) + } + if err != nil { + break // EOF + } + } + } +} + +// modify link & image tags +func (r *Renderer) modifyHTMLTags(source []byte, target io.Writer) (bool, error) { + modified := false + z := html.NewTokenizer(bytes.NewReader(source)) + for { + tt := z.Next() + if tt == html.ErrorToken { + return modified, nil // end of tokens + } + t := z.Token() + if "a" == t.Data { + for i, a := range t.Attr { + if a.Key == "href" { + dest, err := r.linkResolver(a.Val, false) + if err != nil { + return modified, err + } + if a.Val != dest { + t.Attr[i].Val = dest + modified = true + } + break + } + } + } else if "img" == t.Data { + for i, a := range t.Attr { + if a.Key == "src" { + dest, err := r.linkResolver(a.Val, true) + if err != nil { + return modified, err + } + if a.Val != dest { + t.Attr[i].Val = dest + modified = true + } + break + } + } + } + _, _ = target.Write([]byte(t.String())) + } +} + +func (r *Renderer) calcEmphasisChar(n ast.Node) (ch byte, txt []byte) { + ch = '*' // default char + // check if first emphasis child determines the char + if n.Kind() == ast.KindEmphasis && n.FirstChild() != nil && n.FirstChild().Kind() == ast.KindEmphasis { + if n.(*ast.Emphasis).Level == 1 && n.FirstChild().(*ast.Emphasis).Level == 1 { + cch, _ := r.calcEmphasisChar(n.FirstChild()) + if cch == '*' { + ch = '_' // handle nested case + } + return + } + } + // get node text + for c := n.FirstChild(); c != nil; c = c.NextSibling() { + switch c.Kind() { + case ast.KindText: + txt = append(txt, c.Text(r.source)...) + case ast.KindEmphasis: // skip + default: + _, t := r.calcEmphasisChar(c) + txt = append(txt, t...) + } + } + // determine char + if n.Kind() == ast.KindEmphasis { + for i, b := range txt { + if b == '*' { + if i-1 >= 0 && txt[i-1] == '\\' { + continue + } + ch = '_' // unescaped asterisk -> switch to underscore + break + } + } + } + return +} + +// if text could become different AST element additional indents must be added +func (r *Renderer) additionalIndents(text []byte, n *ast.Text) { + // if there is a new line or intended new line before the text + if n.PreviousSibling() != nil && len(text) > 0 { + idx := bytes.LastIndexByte(r.writer.Bytes(), '\n') + if idx == -1 { + idx = 0 + } + last := r.writer.Bytes()[idx:] + if len(last) > 0 && (last[len(last)-1] == '\n' || bytes.Equal(last[1:], r.indents)) { + var addIndents bool + switch text[0] { + case '-', '+', '*', '#', '=': + addIndents = true + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + addIndents = marker.Match(text) + } + if addIndents { + _, _ = r.writer.Write([]byte(" ")) + } + } + } +} + +// sets blank previous line if needed +func breakBlockquoteLazyContinuation(n ast.Node) { + if n != nil && !n.HasBlankPreviousLines() { + if n.Kind() == ast.KindParagraph || n.Kind() == ast.KindTextBlock { + n.SetBlankPreviousLines(true) + } + } +} + +// build ListItem marker +func buildListMarker(n *ast.ListItem) []byte { + p := n.Parent().(*ast.List) + if p.IsOrdered() { + return []byte(fmt.Sprintf("%d%c ", p.Start, p.Marker)) + } + return []byte{p.Marker, ' '} +} + +func isClassicAutolink(link []byte, cnt []byte) bool { + if containsTrailingPunctuation(link[len(link)-1]) { + return true + } + if len(cnt) > 0 && !isDelimiter(cnt[len(cnt)-1]) { + return true + } + if !bytes.Equal(link, util.URLEscape(link, false)) { + return true + } + if http.Match(link) { + return false // GFM autolink http/s + } + if www.Match(link) { + return false // GFM autolink www + } + if email.Match(link) { + return false // GFM autolink email + } + return true +} + +func isDelimiter(b byte) bool { + switch b { + case '\n', '\r', '\t', '\f', '\v', '\x85', '\xa0', ' ', '*', '_', '~', '(': + { + return true + } + default: + return false + } +} + +func containsTrailingPunctuation(b byte) bool { + switch b { + case '?', '!', '.', ',', ':', '*', '_', '~': + { + return true + } + default: + return false + } +} + +func getLinkTitleWrapper(title []byte) byte { + var dq, sq bool + for i, b := range title { + if b == '"' { + if i-1 >= 0 && title[i-1] == '\\' { + continue + } + dq = true + } else if b == '\'' { + if i-1 >= 0 && title[i-1] == '\\' { + continue + } + sq = true + } + } + if dq && sq { + return '(' + } else if dq { + return '\'' + } + return '"' +} + +func wrapLinkDestination(dest []byte) bool { + var lp, elp int + var rp, erp int + for i, b := range dest { + // check for controls & space + if b <= '\x1f' || b == '\x7f' || b == '\x20' { + return true + } + if b == '(' { + lp++ + if i-1 >= 0 && dest[i-1] == '\\' { + elp++ + } + } else if b == ')' { + rp++ + if i-1 >= 0 && dest[i-1] == '\\' { + erp++ + } + } + } + // check parentheses + return (lp-elp)-(rp-elp) != 0 +} + +func nextIsLineBreak(next ast.Node, source []byte) bool { + if next != nil && next.Kind() == ast.KindText { + n := next.(*ast.Text) + if n.HardLineBreak() || n.SoftLineBreak() { + cnt := n.Text(source) + cnt = bytes.TrimSpace(cnt) + return len(cnt) == 0 + } + } + return false +} + +// escape pipes in code span when table scope +func escapePipes(t []byte) []byte { + var et []byte + idx := 0 + for i, b := range t { + if b == '|' { + if i > 0 && t[i-1] == '\\' { + continue + } + if i == 0 { + et = append(et, '\\', '|') + } else { + et = append(et, t[idx:i]...) + et = append(et, '\\', '|') + } + idx = i + 1 + } + } + if idx > 0 { + et = append(et, t[idx:]...) + return et + } + return t +} diff --git a/pkg/markdown/link_modifier_test.go b/pkg/markdown/link_modifier_test.go new file mode 100644 index 00000000..42873e93 --- /dev/null +++ b/pkg/markdown/link_modifier_test.go @@ -0,0 +1,220 @@ +// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package markdown_test + +import ( + "bytes" + "errors" + "github.com/gardener/docforge/pkg/markdown" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/renderer" +) + +var _ = Describe("Links modifier", func() { + var ( + lr *linkResolver + rnd renderer.Renderer + md string + doc ast.Node + err error + buf *bytes.Buffer + exp string + ) + BeforeEach(func() { + lr = &linkResolver{} + rnd = markdown.NewLinkModifierRenderer(markdown.WithLinkResolver(lr.fakeLink)) + md = "## Heading level 2\n\nI really like using Markdown.\n" + exp = md + }) + JustBeforeEach(func() { + doc, err = markdown.Parse([]byte(md)) + Expect(err).NotTo(HaveOccurred()) + Expect(doc).NotTo(BeNil()) + buf = &bytes.Buffer{} + err = rnd.Render(buf, []byte(md), doc) + }) + When("Render markdown", func() { + It("renders the markdown successfully", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Bytes()).To(Equal([]byte(exp))) + }) + }) + When("Render markdown with auto links", func() { + Context("email autolink", func() { + BeforeEach(func() { + md = "mails:\nfoo@bar.baz\n\n" + exp = md + }) + It("does not modify email", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Bytes()).To(Equal([]byte(exp))) + }) + }) + Context("URL autolink", func() { + BeforeEach(func() { + lr.dst = "https://fake.com" + md = "links:\nhttp://foo.bar.baz\n\n\n(www.google.com/search?q=Markup+(business))\n" + exp = "links:\nhttps://fake.com\n\n\n(https://fake.com)\n" + }) + It("modifies links", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Bytes()).To(Equal([]byte(exp))) + }) + }) + Context("Not an autolink", func() { + BeforeEach(func() { + lr.dst = "https://fake.com" + md = "not links:\n`http://foo.bar.baz`\n```\nhttp://foo.bar.baz\n```\n" + exp = md + }) + It("does not modify code span or code block content", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Bytes()).To(Equal([]byte(exp))) + }) + }) + Context("link resolve error", func() { + BeforeEach(func() { + lr.err = errors.New("fake-error") + md = "link:\nhttp://foo.bar.baz\n" + }) + It("fails to render document", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("fake-error")) + }) + }) + }) + When("Render markdown with links", func() { + BeforeEach(func() { + lr.dst = "https://fake.com" + md = "links:\n[link](/uri \"title\")\n[link](http://example.com?foo=3#frag)\n" + exp = "links:\n[link](https://fake.com \"title\")\n[link](https://fake.com)\n" + }) + It("modifies links", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Bytes()).To(Equal([]byte(exp))) + }) + Context("reference link", func() { + BeforeEach(func() { + md = "link:\n[foo][bar]\n\n[bar]: /url \"title\"\n" + exp = "link:\n[foo](https://fake.com \"title\")\n\n" + }) + It("modifies reference link", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Bytes()).To(Equal([]byte(exp))) + }) + }) + Context("link resolve error", func() { + BeforeEach(func() { + lr.err = errors.New("fake-error") + }) + It("fails to render document", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("fake-error")) + }) + }) + }) + When("Render markdown with images", func() { + BeforeEach(func() { + lr.dst = "https://fake.com" + md = "images:\n![foo](/url \"title\")\n![foo [bar](/url)](/url2)\n" + exp = "images:\n![foo](https://fake.com \"title\")\n![foo [bar](https://fake.com)](https://fake.com)\n" + }) + It("modifies images", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Bytes()).To(Equal([]byte(exp))) + }) + Context("reference image", func() { + BeforeEach(func() { + md = "image:\n![foo][bar]\n\n[bar]: /url\n" + exp = "image:\n![foo](https://fake.com)\n\n" + }) + It("modifies reference image", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Bytes()).To(Equal([]byte(exp))) + }) + }) + Context("image resolve error", func() { + BeforeEach(func() { + lr.err = errors.New("fake-error") + }) + It("fails to render document", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("fake-error")) + }) + }) + }) + When("Render markdown with HTML links", func() { + BeforeEach(func() { + lr.dst = "https://fake.com" + md = "block:\n

\nbaz\n

\n\nrow:\nfoo \n" + exp = "block:\n

\nbaz\n

\n\nrow:\nfoo \n" + }) + It("modifies HTML links", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Bytes()).To(Equal([]byte(exp))) + }) + Context("links in comments", func() { + BeforeEach(func() { + md = "block:\n\nrow:\nfoo \n" + exp = md + }) + It("does not modify the links", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Bytes()).To(Equal([]byte(exp))) + }) + }) + Context("link resolve error", func() { + BeforeEach(func() { + lr.err = errors.New("fake-error") + }) + It("fails to render document", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("fake-error")) + }) + }) + }) + When("Render markdown with HTML images", func() { + BeforeEach(func() { + lr.dst = "https://fake.com" + md = "block:\n

\n\"bar\"\n

\n\nrow:\nfoo \"baz\"/\n" + exp = "block:\n

\n\"bar\"\n

\n\nrow:\nfoo \"baz\"/\n" + }) + It("modifies HTML links", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Bytes()).To(Equal([]byte(exp))) + }) + Context("images in comments", func() { + BeforeEach(func() { + md = "block:\n\n\nrow:\nfoo \n" + exp = md + }) + It("does not modify the images", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(buf.Bytes()).To(Equal([]byte(exp))) + }) + }) + Context("image resolve error", func() { + BeforeEach(func() { + lr.err = errors.New("fake-error") + }) + It("fails to render document", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("fake-error")) + }) + }) + }) +}) + +type linkResolver struct { + dst string + err error +} + +// implements markdown.ResolveLink and fakes the result +func (lr *linkResolver) fakeLink(_ string, _ bool) (string, error) { + return lr.dst, lr.err +} diff --git a/pkg/markdown/links.go b/pkg/markdown/links.go deleted file mode 100644 index 182987ec..00000000 --- a/pkg/markdown/links.go +++ /dev/null @@ -1,103 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package markdown - -import ( - "bytes" - "fmt" - - "github.com/gardener/docforge/pkg/markdown/parser" - // "github.com/gomarkdown/markdown" - // "github.com/gomarkdown/markdown/ast" - // "github.com/gomarkdown/markdown/parser" -) - -const ( - // extensions = parser.CommonExtensions | parser.AutoHeadingIDs - - // Link is a link markdown type - Link Type = iota - // Image is an image markdown type - Image -) - -// Type is an enumeration for markdown types -type Type int - -func (m Type) String() string { - return [...]string{"link", "image"}[m] -} - -// NewType creates a markdown Type enum from string -func NewType(markdownTypeString string) (Type, error) { - switch markdownTypeString { - case "link": - return Link, nil - case "image": - return Image, nil - } - return 0, fmt.Errorf("Unknown markdown type string '%s'. Must be one of %v", markdownTypeString, []string{"link", "image"}) -} - -// UpdateMarkdownLink is a callback function invoked on each link -// by mardown#UpdateMarkdownLinks -// It is supplied link attributes and is expected to return them, potentially -// updated, or error. -// A nil destination will yield removing of this link/image markup, -// leaving only the text component if it's a link -// Nil text or title returned yield no change. Any other value replaces -// the original. If a returned title is empty string an originally -// existing title element will be completely removed -type UpdateMarkdownLink func(markdownType Type, destination, text, title []byte) ([]byte, []byte, []byte, error) - -// UpdateMarkdownLinks changes document links destinations, consulting -// with callback on the destination to use on each link or image in document. -// If a callback returns "" for a destination, this is interpreted as -// request to remove the link destination and leave only the link text or in -// case it's an image - to remove it completely. -// TODO: failfast vs fault tolerance support? -func UpdateMarkdownLinks(document parser.Document, callback UpdateMarkdownLink) ([]byte, error) { - if callback == nil { - return nil, nil - } - err := document.ListLinks(func(l parser.Link) (parser.Link, error) { - var ( - destination, text, title []byte - err error - t Type - ) - if l.IsImage() { - t = Image - } else { - t = Link - } - text = l.GetText() - if destination, text, title, err = callback(t, l.GetDestination(), text, l.GetTitle()); err != nil { - return nil, err - } - return updateLink(l, destination, text, title), nil - }) - if err != nil { - return nil, err - } - return document.Bytes(), err -} - -func updateLink(link parser.Link, destination, text, title []byte) parser.Link { - if destination == nil { - link.Remove(text != nil && len(text) > 0) - return nil - } - if text != nil && !bytes.Equal(link.GetText(), text) { - link.SetText(text) - } - if destination != nil && !bytes.Equal(link.GetDestination(), destination) { - link.SetDestination(destination) - } - if title != nil && !bytes.Equal(link.GetTitle(), title) { - link.SetTitle(title) - } - return link -} diff --git a/pkg/markdown/links_test.go b/pkg/markdown/links_test.go deleted file mode 100644 index 55c4aaaa..00000000 --- a/pkg/markdown/links_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package markdown - -import ( - "testing" - - "github.com/gardener/docforge/pkg/markdown/parser" - "github.com/stretchr/testify/assert" -) - -func TestUpdateMarkdownLinks(t *testing.T) { - testCases := []struct { - in []byte - cb UpdateMarkdownLink - wantBlob []byte - wantErr error - }{ - { - []byte(`[abc](https://abc.xyz/a/b/c.md)`), - func(markdownType Type, destination, text, title []byte) ([]byte, []byte, []byte, error) { - return []byte("https://abc.xyz/a/b/c/d/e/f/g.md"), []byte("abcdefg"), nil, nil - }, - []byte(`[abcdefg](https://abc.xyz/a/b/c/d/e/f/g.md)`), - nil, - }, - { - []byte(`[abcdefg](https://abc.xyz/a/b/c/d/e/f/g.md)`), - func(markdownType Type, destination, text, title []byte) ([]byte, []byte, []byte, error) { - return []byte("https://abc.xyz/a/b/c.md"), []byte("abc"), nil, nil - }, - []byte(`[abc](https://abc.xyz/a/b/c.md)`), - nil, - }, - { - []byte(`[abc](https://abc.xyz/a/b/c.md)`), - func(markdownType Type, destination, text, title []byte) ([]byte, []byte, []byte, error) { - return []byte("https://abc.xyz/a/b/c.md"), []byte("abc"), nil, nil - }, - []byte(`[abc](https://abc.xyz/a/b/c.md)`), - nil, - }, - { - []byte(`[abc](https://abc.xyz/a/b/c.md)`), - func(markdownType Type, destination, text, title []byte) ([]byte, []byte, []byte, error) { - return []byte("https://abc.xyz/a/b/d.md"), []byte("abd"), nil, nil - }, - []byte(`[abd](https://abc.xyz/a/b/d.md)`), - nil, - }, - } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - - p := parser.NewParser() - document := p.Parse(tc.in) - gotBlob, gotErr := UpdateMarkdownLinks(document, tc.cb) - assert.Equal(t, string(tc.wantBlob), string(gotBlob)) - if tc.wantErr != nil { - assert.Error(t, gotErr) - } else { - assert.Nil(t, gotErr) - } - }) - } -} diff --git a/pkg/markdown/markdown_suite_test.go b/pkg/markdown/markdown_suite_test.go new file mode 100644 index 00000000..8688eb01 --- /dev/null +++ b/pkg/markdown/markdown_suite_test.go @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package markdown_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "testing" +) + +func TestMarkdown(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Markdown Suite") +} diff --git a/pkg/markdown/parser.go b/pkg/markdown/parser.go new file mode 100644 index 00000000..a215c40d --- /dev/null +++ b/pkg/markdown/parser.go @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package markdown + +import ( + "github.com/yuin/goldmark" + meta "github.com/yuin/goldmark-meta" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" +) + +var ( + // parser extension for GitHub Flavored Markdown & Frontmatter support + extensions = []goldmark.Extender{ + extension.GFM, + meta.Meta, + } + // goldmark.Markdown parser with GFM extensions + gmParser = goldmark.New(goldmark.WithExtensions(extensions...)) +) + +// Parse markdown content and returns AST node or error +func Parse(source []byte) (ast.Node, error) { + reader := text.NewReader(source) + context := parser.NewContext() + doc := gmParser.Parser().Parse(reader, parser.WithContext(context)) + fmb, err := meta.TryGet(context) + if err != nil { + return nil, err + } + if doc.Kind() == ast.KindDocument { + doc.(*ast.Document).SetMeta(fmb) + } + return doc, nil +} diff --git a/pkg/markdown/parser/autolinks_parse.go b/pkg/markdown/parser/autolinks_parse.go deleted file mode 100644 index 356b5df5..00000000 --- a/pkg/markdown/parser/autolinks_parse.go +++ /dev/null @@ -1,475 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package parser - -import ( - "bytes" - "regexp" -) - -// autolinkType specifies a kind of autolink that gets detected. -type autolinkType int - -// These are the possible flag values for the autolink renderer. -const ( - notAutolink autolinkType = iota - normalAutolink - emailAutolink -) - -var ( - urlRe = `((https?|ftp):\/\/|\/)[-A-Za-z0-9+&@#\/%?=~_|!:,.;\(\)]+` - anchorRe = regexp.MustCompile(`^(]+")?\s?>` + urlRe + `<\/a>)`) - - // TODO: improve this regexp to catch all possible entities: - htmlEntityRe = regexp.MustCompile(`&[a-z]{2,5};`) -) - -// '<' when tags or autolinks are allowed -func parseLeftAngle(p *parser, data []byte, offset int) (int, Link) { - data = data[offset:] - start := offset - altype, end := tagLength(data) - // if size := p.inlineHTMLComment(data); size > 0 { - // end = size - // } - if end <= 2 { - return end, nil - } - if altype == notAutolink { - // htmlTag := &ast.HTMLSpan{} - // htmlTag.Literal = data[:end] - return end, nil - } - - var uLink bytes.Buffer - unescapeText(&uLink, data[1:end+1-2]) - if uLink.Len() <= 0 { - return end, nil - } - // link := uLink.Bytes() - node := &link{ - // uLink.Bytes(), - start: start, - end: offset + end, - destination: &bytesRange{ - start: start + 1, - end: offset + end - 1, - }, - linkType: linkAuto, - } - // if altype == emailAutolink { - // node.Destination = append([]byte("mailto:"), link...) - // } - // link = stripMailto(link) - return end, node -} - -// '\\' backslash escape -var escapeChars = []byte("\\`*_{}[]()#+-.!:|&<>~^") - -func escape(p *Parser, data []byte, offset int) (int, []byte) { - data = data[offset:] - - if len(data) <= 1 { - return 2, nil - } - - // if p.extensions&NonBlockingSpace != 0 && data[1] == ' ' { - // return 2, &ast.NonBlockingSpace{} - // } - - // if p.extensions&BackslashLineBreak != 0 && data[1] == '\n' { - // return 2, &ast.Hardbreak{} - // } - - if bytes.IndexByte(escapeChars, data[1]) < 0 { - return 0, nil - } - - return 2, data[1:2] -} - -// '&' escaped when it doesn't belong to an entity -// valid entities are assumed to be anything matching &#?[A-Za-z0-9]+; -func entity(p *Parser, data []byte, offset int) (int, []byte) { - data = data[offset:] - - end := skipCharN(data, 1, '#', 1) - end = skipAlnum(data, end) - - if end < len(data) && data[end] == ';' { - end++ // real entity - } else { - return 0, nil // lone '&' - } - - ent := data[:end] - // undo & escaping or it will be converted to &amp; by another - // escaper in the renderer - if bytes.Equal(ent, []byte("&")) { - ent = []byte{'&'} - } - - return end, ent -} - -func linkEndsWithEntity(data []byte, linkEnd int) bool { - entityRanges := htmlEntityRe.FindAllIndex(data[:linkEnd], -1) - return entityRanges != nil && entityRanges[len(entityRanges)-1][1] == linkEnd -} - -// hasPrefixCaseInsensitive is a custom implementation of -// strings.HasPrefix(strings.ToLower(s), prefix) -// we rolled our own because ToLower pulls in a huge machinery of lowercasing -// anything from Unicode and that's very slow. Since this func will only be -// used on ASCII protocol prefixes, we can take shortcuts. -func hasPrefixCaseInsensitive(s, prefix []byte) bool { - if len(s) < len(prefix) { - return false - } - delta := byte('a' - 'A') - for i, b := range prefix { - if b != s[i] && b != s[i]+delta { - return false - } - } - return true -} - -var protocolPrefixes = [][]byte{ - []byte("http://"), - []byte("https://"), - []byte("ftp://"), - []byte("file://"), - []byte("mailto:"), -} - -const shortestPrefix = 6 // len("ftp://"), the shortest of the above - -func maybeAutoLink(p *parser, data []byte, offset int) (int, Link) { - // quick check to rule out most false hits - if p.insideLink || len(data) < offset+shortestPrefix { - return 0, nil - } - for _, prefix := range protocolPrefixes { - endOfHead := offset + 8 // 8 is the len() of the longest prefix - if endOfHead > len(data) { - endOfHead = len(data) - } - if hasPrefixCaseInsensitive(data[offset:endOfHead], prefix) { - return parseAutoLink(p, data, offset) - } - } - return 0, nil -} - -func codeSpan(p *parser, data []byte, offset int) (int, Link) { - var backtickCount, i, end int - data = data[offset:] - - for backtickCount < len(data) && data[backtickCount] == '`' { - backtickCount++ - } - - for end = backtickCount; end < len(data) && i < backtickCount; end++ { - if data[end] == '`' { - i++ - } else { - i = 0 - } - } - - // no matching delimiter? - if i < backtickCount && end >= len(data) { - return 0, nil - } - - // trim outside whitespace - fBegin := backtickCount - for fBegin < end && data[fBegin] == ' ' { - fBegin++ - } - - fEnd := end - backtickCount - for fEnd > fBegin && data[fEnd-1] == ' ' { - fEnd-- - } - return end, nil -} - -func parseAutoLink(p *parser, data []byte, offset int) (int, Link) { - // Now a more expensive check to see if we're not inside an anchor element - anchorStart := offset - offsetFromAnchor := 0 - for anchorStart > 0 && data[anchorStart] != '<' { - anchorStart-- - offsetFromAnchor++ - } - - anchorStr := anchorRe.Find(data[anchorStart:]) - if anchorStr != nil { - // anchorClose := &ast.HTMLSpan{} - // anchorClose.Literal = anchorStr[offsetFromAnchor:] - return len(anchorStr) - offsetFromAnchor, nil - } - - // scan backward for a word boundary - rewind := 0 - for offset-rewind > 0 && rewind <= 7 && isLetter(data[offset-rewind-1]) { - rewind++ - } - if rewind > 6 { // longest supported protocol is "mailto" which has 6 letters - return 0, nil - } - - origData := data - data = data[offset-rewind:] - - if !isSafeLink(data) { - return 0, nil - } - - linkB := offset - rewind - linkEnd := 0 - for linkEnd < len(data) && !isEndOfLink(data[linkEnd]) { - linkEnd++ - } - - // Skip punctuation at the end of the link - if (data[linkEnd-1] == '.' || data[linkEnd-1] == ',' || data[linkEnd-1] == '?' || data[linkEnd-1] == '!') && data[linkEnd-2] != '\\' { - linkEnd-- - } - - // But don't skip semicolon if it's a part of escaped entity: - if data[linkEnd-1] == ';' && data[linkEnd-2] != '\\' && !linkEndsWithEntity(data, linkEnd) { - linkEnd-- - } - - // See if the link finishes with a punctuation sign that can be closed. - var copen byte - switch data[linkEnd-1] { - case '"': - copen = '"' - case '\'': - copen = '\'' - case ')': - copen = '(' - case ']': - copen = '[' - case '}': - copen = '{' - default: - copen = 0 - } - - if copen != 0 { - bufEnd := offset - rewind + linkEnd - 2 - - openDelim := 1 - - /* Try to close the final punctuation sign in this same line; - * if we managed to close it outside of the URL, that means that it's - * not part of the URL. If it closes inside the URL, that means it - * is part of the URL. - * - * Examples: - * - * foo http://www.pokemon.com/Pikachu_(Electric) bar - * => http://www.pokemon.com/Pikachu_(Electric) - * - * foo (http://www.pokemon.com/Pikachu_(Electric)) bar - * => http://www.pokemon.com/Pikachu_(Electric) - * - * foo http://www.pokemon.com/Pikachu_(Electric)) bar - * => http://www.pokemon.com/Pikachu_(Electric)) - * - * (foo http://www.pokemon.com/Pikachu_(Electric)) bar - * => foo http://www.pokemon.com/Pikachu_(Electric) - */ - - for bufEnd >= 0 && origData[bufEnd] != '\n' && openDelim != 0 { - if origData[bufEnd] == data[linkEnd-1] { - openDelim++ - } - - if origData[bufEnd] == copen { - openDelim-- - } - - bufEnd-- - } - - if openDelim == 0 { - linkEnd-- - } - } - - for ; data[linkEnd-1] == '*' || data[linkEnd-1] == '_' || data[linkEnd-1] == '`'; linkEnd-- { - } - - var uLink bytes.Buffer - unescapeText(&uLink, data[:linkEnd]) - - if uLink.Len() > 0 { - node := &link{ - // uLink.Bytes(), - start: linkB, - end: offset + linkEnd, - destination: &bytesRange{ - start: linkB, - end: offset + linkEnd, - }, - linkType: linkAuto, - } - return linkEnd, node - } - - return linkEnd, nil -} - -func isEndOfLink(char byte) bool { - return isSpace(char) || char == '<' -} - -var validUris = [][]byte{[]byte("http://"), []byte("https://"), []byte("ftp://"), []byte("mailto://")} -var validPaths = [][]byte{[]byte("/"), []byte("./"), []byte("../")} - -func isSafeLink(link []byte) bool { - nLink := len(link) - for _, path := range validPaths { - nPath := len(path) - linkPrefix := link[:nPath] - if nLink >= nPath && bytes.Equal(linkPrefix, path) { - if nLink == nPath { - return true - } else if isAlnum(link[nPath]) { - return true - } - } - } - - for _, prefix := range validUris { - // TODO: handle unicode here - // case-insensitive prefix test - nPrefix := len(prefix) - if nLink > nPrefix { - linkPrefix := bytes.ToLower(link[:nPrefix]) - if bytes.Equal(linkPrefix, prefix) && isAlnum(link[nPrefix]) { - return true - } - } - } - - return false -} - -// return the length of the given tag, or 0 is it's not valid -func tagLength(data []byte) (autolink autolinkType, end int) { - var i, j int - - // a valid tag can't be shorter than 3 chars - if len(data) < 3 { - return notAutolink, 0 - } - - // begins with a '<' optionally followed by '/', followed by letter or number - if data[0] != '<' { - return notAutolink, 0 - } - if data[1] == '/' { - i = 2 - } else { - i = 1 - } - - if !isAlnum(data[i]) { - return notAutolink, 0 - } - - // scheme test - autolink = notAutolink - - // try to find the beginning of an URI - for i < len(data) && (isAlnum(data[i]) || data[i] == '.' || data[i] == '+' || data[i] == '-') { - i++ - } - - if i > 1 && i < len(data) && data[i] == '@' { - if j = isMailtoAutoLink(data[i:]); j != 0 { - return emailAutolink, i + j - } - } - - if i > 2 && i < len(data) && data[i] == ':' { - autolink = normalAutolink - i++ - } - - // complete autolink test: no whitespace or ' or " - switch { - case i >= len(data): - autolink = notAutolink - case autolink != notAutolink: - j = i - - for i < len(data) { - if data[i] == '\\' { - i += 2 - } else if data[i] == '>' || data[i] == '\'' || data[i] == '"' || isSpace(data[i]) { - break - } else { - i++ - } - - } - - if i >= len(data) { - return autolink, 0 - } - if i > j && data[i] == '>' { - return autolink, i + 1 - } - - // one of the forbidden chars has been found - autolink = notAutolink - } - i += bytes.IndexByte(data[i:], '>') - if i < 0 { - return autolink, 0 - } - return autolink, i + 1 -} - -// look for the address part of a mail autolink and '>' -// this is less strict than the original markdown e-mail address matching -func isMailtoAutoLink(data []byte) int { - nb := 0 - - // address is assumed to be: [-@._a-zA-Z0-9]+ with exactly one '@' - for i, c := range data { - if isAlnum(c) { - continue - } - - switch c { - case '@': - nb++ - - case '-', '.', '_': - break - - case '>': - if nb == 1 { - return i + 1 - } - return 0 - default: - return 0 - } - } - - return 0 -} diff --git a/pkg/markdown/parser/autolinks_parse_test.go b/pkg/markdown/parser/autolinks_parse_test.go deleted file mode 100644 index b544a5b3..00000000 --- a/pkg/markdown/parser/autolinks_parse_test.go +++ /dev/null @@ -1,244 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package parser - -import ( - "fmt" - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestParseAutoLinks(t *testing.T) { - testCases := []struct { - in string - // 0: link start - // 1: link end - want [][]int - }{ - { - ` `, - [][]int{ - []int{2, 17}, - []int{3, 16}, - }, - }, - { - ` . `, - [][]int{ - []int{2, 17}, - []int{3, 16}, - }, - }, - { - ` . `, - [][]int{ - []int{2, 27}, - []int{3, 26}, - }, - }, - { - ` `, - nil, - }, - { - ` <./a.com> `, - nil, - }, - { - ` < https://a.com > `, - nil, - }, - { - ` `, - [][]int{ - []int{2, 23}, - []int{3, 22}, - }, - }, - { - ` `, - [][]int{ - []int{2, 14}, - []int{3, 13}, - }, - }, - { - ` < mailto://a@mail.com > `, - nil, - }, - } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - p := NewParser() - var want *link - if tc.want != nil { - want = &link{ - start: tc.want[0][0], - end: tc.want[0][1], - destination: &bytesRange{ - start: tc.want[1][0], - end: tc.want[1][1], - }, - linkType: linkAuto, - } - } - s := strings.Split(tc.in, " ") - offset := 0 - for _, w := range s { - if len(w) > 0 { - break - } - offset++ - } - _, got := parseLeftAngle(p.(*parser), []byte(tc.in), offset) - if tc.want == nil && assert.Nil(t, got) { - return - } - if assert.Equal(t, want, got) { - if got == nil { - fmt.Println("|nil|") - } else { - l := got.(*link) - fmt.Printf("|%s|\n", string([]byte(tc.in)[l.start:l.end])) - } - } - }) - } -} - -func TestParseAutoLinksExtended(t *testing.T) { - testCases := []struct { - in string - // 0: link start - // 1: link end - want []int - }{ - { - ` https://a.com `, - []int{2, 15}, - }, - { - ` https://a.com. `, - []int{2, 15}, - }, - { - ` https://a.com#q?a=b&c=3. `, - []int{2, 25}, - }, - { - ` ./a.com `, - []int{2, 9}, - }, - { - ` www.a.com `, - nil, - }, - { - ` a.com `, - nil, - }, - { - " https:\n//a.com ", - nil, - }, - { - ` (https://a.com?a=b). `, - nil, - }, - { - ` mailto://a@mail.com `, - []int{2, 21}, - }, - { - ` a@mail.com `, - nil, - }, - } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - p := NewParser() - var want *link - if tc.want != nil { - want = &link{ - start: tc.want[0], - end: tc.want[1], - destination: &bytesRange{ - start: tc.want[0], - end: tc.want[1], - }, - linkType: linkAuto, - } - } - s := strings.Split(tc.in, " ") - offset := 0 - for _, w := range s { - if len(w) > 0 { - break - } - offset++ - } - data := []byte(tc.in) - _, got := parseAutoLink(p.(*parser), data, offset) - if tc.want == nil && assert.Nil(t, got) { - return - } - if assert.Equal(t, want, got) { - if got == nil { - fmt.Println("|nil|") - } else { - l := got.(*link) - fmt.Printf("|%s|\n", string([]byte(tc.in)[l.start:l.end])) - } - } - }) - } -} - -func Test_codeSpan(t *testing.T) { - type args struct { - data string - offset int - } - tests := []struct { - name string - args args - wantConsumed int - }{ - { - name: "single_backtick_tuple", - args: args{ - "Configure the client to have a callback url of `http://localhost:8000`.", - 47, - }, - wantConsumed: 23, - }, - { - name: "tripple_backtick_tuple", - args: args{ - "Configure the client to have a callback url of ```http://localhost:8000```.", - 47, - }, - wantConsumed: 27, - }, - { - name: "zero_when_closing_backtick_is_missing", - args: args{ - "`callback url of.", - 0, - }, - wantConsumed: 0, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - consumed, _ := codeSpan(nil, []byte(tt.args.data), tt.args.offset) - if consumed != tt.wantConsumed { - t.Errorf("codeSpan() got = %v, want %v", consumed, tt.wantConsumed) - } - }) - } -} diff --git a/pkg/markdown/parser/charscan.go b/pkg/markdown/parser/charscan.go deleted file mode 100644 index 9742123f..00000000 --- a/pkg/markdown/parser/charscan.go +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package parser - -import "bytes" - -// like skipChar but only skips up to max characters -func skipCharN(data []byte, i int, c byte, max int) int { - n := len(data) - for i < n && max > 0 && data[i] == c { - i++ - max-- - } - return i -} - -func skipAlnum(data []byte, i int) int { - n := len(data) - for i < n && isAlnum(data[i]) { - i++ - } - return i -} - -func skipSpace(data []byte, i int) int { - n := len(data) - for i < n && isSpace(data[i]) { - i++ - } - return i -} - -// skipUntilChar advances i as long as data[i] != c -func skipUntilChar(data []byte, i int, c byte) int { - n := len(data) - for i < n && data[i] != c { - i++ - } - return i -} - -// skipUntilCharBackwards traces back i as long as data[i] != c -func skipUntilCharBackwards(data []byte, i int, c byte) int { - for i >= 0 && data[i] != c { - i-- - } - return i -} - -// isSpace returns true if c is a white-space character -func isSpace(c byte) bool { - return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v' -} - -// isLetter returns true if c is ascii letter -func isLetter(c byte) bool { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') -} - -// isAlnum returns true if c is a digit or letter -// TODO: check when this is looking for ASCII alnum and when it should use unicode -func isAlnum(c byte) bool { - return (c >= '0' && c <= '9') || isLetter(c) -} - -func unescapeText(ob *bytes.Buffer, src []byte) { - i := 0 - for i < len(src) { - org := i - for i < len(src) && src[i] != '\\' { - i++ - } - - if i > org { - ob.Write(src[org:i]) - } - - if i+1 >= len(src) { - break - } - - ob.WriteByte(src[i+1]) - i += 2 - } -} diff --git a/pkg/markdown/parser/document.go b/pkg/markdown/parser/document.go deleted file mode 100644 index a502653b..00000000 --- a/pkg/markdown/parser/document.go +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package parser - -type document struct { - data []byte - links []Link -} - -func (d *document) ListLinks(cb OnLinkListed) error { - if cb != nil && d.links != nil { - i := 0 - for _, l := range d.links { - _l, err := cb(l) - if err != nil { - return err - } - if _l != nil { - // Copy over only links that were not removed - d.links[i] = _l - i++ - } - } - // Erase truncated values - for j := i; j < len(d.links); j++ { - d.links[j] = nil - } - d.links = d.links[:i] - } - return nil -} - -func (d *document) Bytes() []byte { - return d.data -} diff --git a/pkg/markdown/parser/links.go b/pkg/markdown/parser/links.go deleted file mode 100644 index 02f7f29d..00000000 --- a/pkg/markdown/parser/links.go +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package parser - -type linkType int - -const ( - linkNormal linkType = iota - linkImg - linkAuto - linkDeferredFootnote - // linkInlineFootnote - // linkCitation -) - -type bytesRange struct { - start int - end int -} - -type link struct { - document *document - start int - end int - text *bytesRange - destination *bytesRange - title *bytesRange - linkType linkType -} - -func offsetSiblingsByteRanges(l *link, offset int) { - var needOffset bool - for _, _l := range l.document.links { - if _l == l { - needOffset = true - continue - } - if needOffset { - offsetLinkByteRanges(_l.(*link), offset) - } - } -} - -func offsetLinkByteRanges(link *link, offset int) { - link.start = offset + link.start - link.end = offset + link.end - if link.text != nil { - link.text.start = offset + link.text.start - link.text.end = offset + link.text.end - } - if link.destination != nil { - link.destination.start = offset + link.destination.start - link.destination.end = offset + link.destination.end - } - if link.title != nil && link.title.start != link.title.end { - link.title.start = offset + link.title.start - link.title.end = offset + link.title.end - } -} - -func replaceBytes(doc []byte, start, end int, text []byte) []byte { - doc1 := doc[:start] - doc2 := doc[end:] - docUpdate := append([]byte{}, doc1...) - docUpdate = append(docUpdate, text...) - docUpdate = append(docUpdate, doc2...) - return docUpdate -} - -func (l *link) SetText(text []byte) { - if l.text == nil { - return - } - offset := len(text) - (l.text.end - l.text.start) - l.document.data = replaceBytes(l.document.data, l.text.start, l.text.end, text) - // offset next link components, if any - if l.destination != nil { - l.destination.start = offset + l.destination.start - l.destination.end = offset + l.destination.end - } - if l.title != nil && l.title.start != l.title.end { - l.title.start = offset + l.title.start - l.title.end = offset + l.title.end - } - offsetSiblingsByteRanges(l, offset) -} - -func (l *link) GetText() []byte { - if l.text == nil { - return nil - } - return l.document.data[l.text.start:l.text.end] -} - -func (l *link) SetDestination(text []byte) { - if l.destination == nil { - return - } - offset := len(text) - (l.destination.end - l.destination.start) - l.document.data = replaceBytes(l.document.data, l.destination.start, l.destination.end, text) - // offset next link components, if any - if l.title != nil && l.title.start != l.title.end { - l.title.start = offset + l.title.start - l.title.end = offset + l.title.end - } - offsetSiblingsByteRanges(l, offset) -} - -func (l *link) GetDestination() []byte { - if l.destination == nil || l.destination.start == l.destination.end { - return nil - } - - return l.document.data[l.destination.start:l.destination.end] -} - -func (l *link) SetTitle(text []byte) { - if l.title == nil { - return - } - offset := len(text) - (l.title.end - l.title.start) - l.document.data = replaceBytes(l.document.data, l.title.start, l.title.end, text) - offsetSiblingsByteRanges(l, offset) -} - -func (l *link) GetTitle() []byte { - if l.title == nil || l.title.start == l.title.end { - return nil - } - - return l.document.data[l.title.start:l.title.end] -} - -func (l *link) Remove(leaveText bool) { - text := []byte("") - if l.linkType != linkImg && leaveText { - text = l.GetText() - } - doc1 := l.document.data[:l.start] - doc2 := l.document.data[l.end:] - l.document.data = append([]byte{}, doc1...) - l.document.data = append(l.document.data, text...) - l.document.data = append(l.document.data, doc2...) - var ( - offset int - _l *link - ok bool - ) - if len(text) > 0 { - offset = len(text) - (l.end - l.start) - } else { - offset = l.start - l.end - } - for i := 0; i < len(l.document.links); i++ { - if _l, ok = l.document.links[i].(*link); !ok { - continue - } - // links positioned after the removed one get offset - if l.end < _l.start { - offsetLinkByteRanges(_l, offset) - } - } -} - -func (l *link) IsImage() bool { - return l.linkType == linkImg -} - -func (l *link) IsAutoLink() bool { - return l.linkType == linkAuto -} - -func (l *link) IsNormalLink() bool { - return l.linkType == linkNormal -} diff --git a/pkg/markdown/parser/links_parse.go b/pkg/markdown/parser/links_parse.go deleted file mode 100644 index 716a5c7a..00000000 --- a/pkg/markdown/parser/links_parse.go +++ /dev/null @@ -1,486 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package parser - -import ( - "bytes" -) - -func maybeImage(p *parser, data []byte, offset int) (int, Link) { - if offset < len(data)-1 && data[offset+1] == '[' { - return parseLink(p, data, offset) - } - return 0, nil -} - -// '[': parse a link or an image or a footnote or a citation -func parseLink(p *parser, data []byte, offset int) (int, Link) { - // no links allowed inside regular links, footnote, and deferred footnotes - if p.insideLink && (offset > 0 && data[offset-1] == '[' || len(data)-1 > offset && data[offset+1] == '^') { - return 0, nil - } - - var t linkType - switch { - // special case: ![^text] == deferred footnote (that follows something with - // an exclamation point) - // case p.extensions&Footnotes != 0 && len(data)-1 > offset && data[offset+1] == '^': - // t = linkDeferredFootnote - // ![alt] == image - case offset >= 0 && data[offset] == '!': - t = linkImg - offset++ - // [@citation], [@-citation], [@?citation], [@!citation] - // case p.extensions&Mmark != 0 && len(data)-1 > offset && data[offset+1] == '@': - // t = linkCitation - // [text] == regular link - // ^[text] == inline footnote - // [^refId] == deferred footnote - // case p.extensions&Footnotes != 0: - // if offset >= 0 && data[offset] == '^' { - // t = linkInlineFootnote - // offset++ - // } else if len(data)-1 > offset && data[offset+1] == '^' { - // t = linkDeferredFootnote - // } - default: - t = linkNormal - } - - data = data[offset:] - - // if t == linkCitation { - // return citation(p, data, 0) - // } - - var ( - i = 1 - // noteID int - /*title,*/ - // link, linkID, altContent []byte - end, linkB, linkE, txtB, txtE, titleB, titleE int - textHasNl = false - ) - start := offset - i = skipSpace(data, i) - txtB = i - - // if t == linkDeferredFootnote { - // i++ - // } - - // look for the matching closing bracket - for level := 1; level > 0 && i < len(data); i++ { - switch { - case data[i] == '\n': - textHasNl = true - - case data[i-1] == '\\': - continue - - case data[i] == '[': - level++ - - case data[i] == ']': - level-- - if level <= 0 { - i-- // compensate for extra i++ in for loop - } - } - } - - if i >= len(data) { - return 0, nil - } - - txtE = i - // remove whitespace at the end of the text - for txtE > txtB && isSpace(data[txtE-1]) { - txtE-- - } - i++ - // var footnoteNode ast.Node - - // skip any amount of whitespace or newline - // (this is much more lax than original markdown syntax) - i = skipSpace(data, i) - - // inline style link - switch { - case i < len(data) && data[i] == '(': - // skip initial whitespace - i++ - - i = skipSpace(data, i) - - linkB = i - brace := 0 - - // look for link end: ' " ) - findlinkend: - for i < len(data) { - switch { - case data[i] == '\\': - i += 2 - - case data[i] == '(': - brace++ - i++ - - case data[i] == ')': - if brace <= 0 { - break findlinkend - } - brace-- - i++ - - case data[i] == '\'' || data[i] == '"': - break findlinkend - - default: - i++ - } - } - if i >= len(data) { - return 0, nil - } - linkE = i - - // look for title end if present - titleB, titleE = 0, 0 - if data[i] == '\'' || data[i] == '"' { - i++ - titleB = i - titleEndCharFound := false - - findtitleend: - for i < len(data) { - switch { - case data[i] == '\\': - i++ - - case data[i] == data[titleB-1]: // matching title delimiter - titleEndCharFound = true - - case titleEndCharFound && data[i] == ')': - end = i - break findtitleend - } - i++ - } - - if i >= len(data) { - return 0, nil - } - - linkE = titleB - 1 - - // skip whitespace after title - titleE = i - 1 - for titleE > titleB && isSpace(data[titleE]) { - titleE-- - } - - // check for closing quote presence - if data[titleE] != '\'' && data[titleE] != '"' { - titleB, titleE = 0, 0 - linkE = i - } - } else { - end = linkE - } - - if len(data) >= end+1 { - end++ - } - - // remove whitespace at the end of the link - for linkE > linkB && isSpace(data[linkE-1]) { - linkE-- - } - - // link whitespace not allowed if not in <> - if data[linkB] != '<' && data[linkE-1] != '>' { - if bytes.ContainsAny(data[linkB:linkE], " ") { - return 0, nil - } - } - - // remove optional angle brackets around the link - if data[linkB] == '<' { - linkB++ - } - if data[linkE-1] == '>' { - linkE-- - } - - // build escaped link and title - // if linkE > linkB { - // link = data[linkB:linkE] - // } - - // if titleE > titleB { - // title = data[titleB:titleE] - // } - - i++ - - // reference style link - // case isReferenceStyleLink(data, i, t): - // var id []byte - // altContentConsidered := false - - // // look for the id - // i++ - // linkB = i - // i = skipUntilChar(data, i, ']') - - // if i >= len(data) { - // return 0, nil - // } - // linkE := i - - // // find the reference - // if linkB == linkE { - // if textHasNl { - // var b bytes.Buffer - - // for j := 1; j < txtE; j++ { - // switch { - // case data[j] != '\n': - // b.WriteByte(data[j]) - // case data[j-1] != ' ': - // b.WriteByte(' ') - // } - // } - - // id = b.Bytes() - // } else { - // id = data[1:txtE] - // altContentConsidered = true - // } - // } else { - // id = data[linkB:linkE] - // } - - // // find the reference with matching id - // lr, ok := p.getRef(string(id)) - // if !ok { - // return 0, nil - // } - - // // keep link and title from reference - // linkID = id - // link = lr.link - // title = lr.title - // if altContentConsidered { - // altContent = lr.text - // } - // i++ - - // shortcut reference style link or reference or inline footnote - default: - // var id []byte - - // craft the id - if textHasNl { - var b bytes.Buffer - - for j := 1; j < txtE; j++ { - switch { - case data[j] != '\n': - b.WriteByte(data[j]) - case data[j-1] != ' ': - b.WriteByte(' ') - } - } - - // id = b.Bytes() - } else { - // if t == linkDeferredFootnote { - // id = data[2:txtE] // get rid of the ^ - // } else { - // id = data[1:txtE] - // } - } - - // footnoteNode = &ast.ListItem{} - // if t == linkInlineFootnote { - // // create a new reference - // noteID = len(p.notes) + 1 - - // var fragment []byte - // if len(id) > 0 { - // if len(id) < 16 { - // fragment = make([]byte, len(id)) - // } else { - // fragment = make([]byte, 16) - // } - // copy(fragment, slugify(id)) - // } else { - // fragment = append([]byte("footnote-"), []byte(strconv.Itoa(noteID))...) - // } - - // ref := &reference{ - // noteID: noteID, - // hasBlock: false, - // link: fragment, - // title: id, - // footnote: footnoteNode, - // } - - // p.notes = append(p.notes, ref) - // p.refsRecord[string(ref.link)] = struct{}{} - - // link = ref.link - // title = ref.title - // } - // } else { - // // find the reference with matching id - // lr, ok := p.getRef(string(id)) - // if !ok { - // return 0, nil - // } - - // if t == linkDeferredFootnote && !p.isFootnote(lr) { - // lr.noteID = len(p.notes) + 1 - // lr.footnote = footnoteNode - // p.notes = append(p.notes, lr) - // p.refsRecord[string(lr.link)] = struct{}{} - // } - - // // keep link and title from reference - // link = lr.link - // // if inline footnote, title == footnote contents - // title = lr.title - // noteID = lr.noteID - // if len(lr.text) > 0 { - // altContent = lr.text - // } - // } - - // rewind the whitespace - i = txtE + 1 - } - - // var uLink []byte - // link = data[linkB:linkE] - // if t == linkNormal || t == linkImg { - // if len(link) > 0 { - // var uLinkBuf bytes.Buffer - // unescapeText(&uLinkBuf, link) - // uLink = uLinkBuf.Bytes() - // } - - // // links need something to click on and somewhere to go - // if len(uLink) == 0 || (t == linkNormal && txtE <= 1) { - // return 0, nil - // } - // } - - // call the relevant rendering function - switch t { - case linkNormal: - if txtE-txtB <= 0 { - return i, nil - } - if linkE-linkB <= 0 { - return i, nil - } - maybeTitle := &bytesRange{ - start: offset + titleB, - end: offset + titleE, - } - if titleB == titleE { - maybeTitle = nil - } - maybeDestination := &bytesRange{ - start: offset + linkB, - end: offset + linkE, - } - if linkB == linkE { - maybeDestination = nil - } - link := &link{ - start: start, - end: offset + end, - text: &bytesRange{ - start: offset + txtB, - end: offset + txtE, - }, - linkType: linkNormal, - // &LiteralComponent{ - // normalizeURI(uLink), - // }, - // title: title, - // DeferredID: linkID, - } - link.title = maybeTitle - link.destination = maybeDestination - // if len(altContent) > 0 { - // ast.AppendChild(link, newTextNode(altContent)) - // } else { - // links cannot contain other links, so turn off link parsing - // temporarily and recurse - insideLink := p.insideLink - p.insideLink = true - p.Parse(data[1:txtE]) - p.insideLink = insideLink - // } - return i, link - - case linkImg: - if linkE-linkB <= 0 { - return i, nil - } - image := &link{ - start: start, - end: offset + end, - text: &bytesRange{ - start: offset + txtB, - end: offset + txtE, - }, - destination: &bytesRange{ - start: offset + linkB, - end: offset + linkE, - }, - title: &bytesRange{ - start: offset + titleB, - end: offset + titleE, - }, - linkType: linkImg, - } - return i + 1, image - - // case linkInlineFootnote, linkDeferredFootnote: - // link := &ast.Link{ - // destination: link, - // title: title, - // NoteID: noteID, - // Footnote: footnoteNode, - // } - // if t == linkDeferredFootnote { - // link.DeferredID = data[2:txtE] - // } - // if t == linkInlineFootnote { - // i++ - // } - // return i, link - - default: - return 0, nil - } -} - -func normalizeURI(s []byte) []byte { - return s // TODO: implement -} - -func isReferenceStyleLink(data []byte, pos int, t linkType) bool { - if t == linkDeferredFootnote { - return false - } - return pos < len(data)-1 && data[pos] == '[' && data[pos+1] != '^' -} diff --git a/pkg/markdown/parser/links_parse_test.go b/pkg/markdown/parser/links_parse_test.go deleted file mode 100644 index a06505d7..00000000 --- a/pkg/markdown/parser/links_parse_test.go +++ /dev/null @@ -1,300 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package parser - -import ( - "fmt" - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestParseLinks(t *testing.T) { - testCases := []struct { - in string - // 1st tuple: link [start:end] - // 2nd tuple: text [start:end] - // 3d tuple: destination [start:end] - // 4th tuple: title [start:end] - want [][]int - }{ - { - `[link](/uri)`, - [][]int{ - []int{0, 12}, - []int{1, 5}, - []int{7, 11}, - }, - }, - { - `[link](/uri "title")`, - [][]int{ - []int{0, 20}, - []int{1, 5}, - []int{7, 11}, - []int{13, 18}, - }, - }, - { - `[link](/uri 'title')`, - [][]int{ - []int{0, 20}, - []int{1, 5}, - []int{7, 11}, - []int{13, 18}, - }, - }, - // { - // // Not supported - // `[link](/uri (title))`, - // nil, - // }, - { - `[link]()`, - nil, - }, - { - `[link](<>)`, - nil, - }, - { - `[link]()`, - [][]int{ - []int{0, 17}, - []int{1, 5}, - []int{8, 15}, - }, - }, - // { - // `[a]()`, - // [][]int{ - // []int{0, 9}, - // []int{1, 2}, - // []int{5, 7}, - // }, - // }, - // { - // `[link](\(foo\))`, - // [][]int{ - // []int{0, 14}, - // []int{1, 5}, - // []int{9, 11}, - // }, - // }, - // { - // `[link](foo(and(bar)))`, - // [][]int{ - // []int{0, 21}, - // []int{1, 5}, - // []int{7, 20}, - // }, - // }, - { - `[link](foo\(and\(bar\))`, - [][]int{ - []int{0, 23}, - []int{1, 5}, - []int{7, 22}, - }, - }, - // { - // `[link]()`, - // [][]int{ - // []int{0, 22}, - // []int{1, 5}, - // []int{7, 21}, - // }, - // }, - { - `[link](foo\)\:)`, - [][]int{ - []int{0, 15}, - []int{1, 5}, - []int{7, 14}, - }, - }, - { - `[link](#fragment)`, - [][]int{ - []int{0, 17}, - []int{1, 5}, - []int{7, 16}, - }, - }, - { - `[link](http://example.com#fragment)`, - [][]int{ - []int{0, 35}, - []int{1, 5}, - []int{7, 34}, - }, - }, - { - `[link](http://example.com?foo=3#frag)`, - [][]int{ - []int{0, 37}, - []int{1, 5}, - []int{7, 36}, - }, - }, - { - `[link](/url 'title "and" title')`, - [][]int{ - []int{0, 32}, - []int{1, 5}, - []int{7, 11}, - []int{13, 30}, - }, - }, - { - ` [a](b.com 'c' ) `, - [][]int{ - []int{2, 19}, - []int{3, 4}, - []int{6, 11}, - []int{15, 16}, - }, - }, - { - ` [a]() `, - [][]int{ - []int{2, 14}, - []int{3, 4}, - []int{7, 12}, - }, - }, - { - ` [a]( 'c') `, - [][]int{ - []int{2, 18}, - []int{3, 4}, - []int{7, 12}, - []int{15, 16}, - }, - }, - { - ` [ a ]( b.com)`, - [][]int{ - []int{2, 16}, - []int{4, 5}, - []int{10, 15}, - }, - }, - { - ` [ a - ]( b.com)`, - [][]int{ - []int{2, 17}, - []int{4, 5}, - []int{11, 16}, - }, - }, - { - `[link]( /uri - "title" )`, - [][]int{ - []int{0, 27}, - []int{1, 5}, - []int{10, 14}, - []int{18, 23}, - }, - }, - { - `[link [foo [bar]]](/uri)`, - [][]int{ - []int{0, 24}, - []int{1, 17}, - []int{19, 23}, - }, - }, - // { - // `[link [bar](/uri)`, - // [][]int{ - // []int{6, 17}, - // []int{7, 10}, - // []int{12, 16}, - // }, - // }, - { - `[link \[bar](/uri)`, - [][]int{ - []int{0, 18}, - []int{1, 11}, - []int{13, 17}, - }, - }, - { - "[link *foo **bar** `#`*](/uri)", - [][]int{ - []int{0, 30}, - []int{1, 23}, - []int{25, 29}, - }, - }, - } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - p := NewParser() - want := &link{ - linkType: linkNormal, - } - for i, tuple := range tc.want { - switch i { - case 0: - { - want.start = tuple[0] - want.end = tuple[1] - } - case 1: - { - want.text = &bytesRange{ - start: tuple[0], - end: tuple[1], - } - } - case 2: - { - want.destination = &bytesRange{ - start: tuple[0], - end: tuple[1], - } - } - case 3: - { - want.title = &bytesRange{ - start: tuple[0], - end: tuple[1], - } - } - } - } - var expected Link = want - zeroValue := &link{} - if *want == *zeroValue { - expected = *new(Link) - } - s := strings.Split(tc.in, " ") - offset := 0 - for _, w := range s { - if len(w) > 0 { - break - } - offset++ - } - - _, got := parseLink(p.(*parser), []byte(tc.in), offset) - - if assert.Equal(t, expected, got) { - if got == nil { - fmt.Println("|nil|") - } else { - l := got.(*link) - fmt.Printf("|%s|\n", string([]byte(tc.in)[l.start:l.end])) - } - } - }) - } -} diff --git a/pkg/markdown/parser/links_test.go b/pkg/markdown/parser/links_test.go deleted file mode 100644 index 28c7dcd6..00000000 --- a/pkg/markdown/parser/links_test.go +++ /dev/null @@ -1,379 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package parser - -import ( - "bytes" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestListLinkRewrites(t *testing.T) { - testCases := []struct { - in *document - listCb OnLinkListed - want []byte - }{ - { - &document{ - []byte("some intro [b](c.com) some text [b1](c1.com)"), - []Link{ - &link{ - start: 11, - end: 21, - text: &bytesRange{ - start: 12, - end: 13, - }, - destination: &bytesRange{ - start: 15, - end: 20, - }, - }, - &link{ - start: 32, - end: 44, - text: &bytesRange{ - start: 33, - end: 35, - }, - destination: &bytesRange{ - start: 37, - end: 43, - }, - }, - }, - }, - func(l Link) (Link, error) { - if bytes.Equal(l.GetText(), []byte("b")) { - l.SetDestination([]byte("e.com")) - l.SetText([]byte("d")) - } - if bytes.Equal(l.GetText(), []byte("b1")) { - l.SetDestination([]byte("c2.com")) - } - return l, nil - }, - []byte("some intro [d](e.com) some text [b1](c2.com)"), - }, - { - &document{ - []byte("some intro [b](c.com)\n`x`some text [b1](c1.com)"), - []Link{ - &link{ - start: 11, - end: 21, - text: &bytesRange{ - start: 12, - end: 13, - }, - destination: &bytesRange{ - start: 15, - end: 20, - }, - }, - &link{ - start: 35, - end: 47, - text: &bytesRange{ - start: 36, - end: 38, - }, - destination: &bytesRange{ - start: 40, - end: 46, - }, - }, - }, - }, - func(l Link) (Link, error) { - if bytes.Equal(l.GetText(), []byte("b")) { - l.SetDestination([]byte("e.com")) - l.SetText([]byte("d")) - } - if bytes.Equal(l.GetText(), []byte("b1")) { - l.SetDestination([]byte("c2.com")) - } - return l, nil - }, - []byte("some intro [d](e.com)\n`x`some text [b1](c2.com)"), - }, - { - &document{ - []byte(`some intro [b](c.com) - some text [b1](c1.com)`), - []Link{ - &link{ - start: 11, - end: 21, - text: &bytesRange{ - start: 12, - end: 13, - }, - destination: &bytesRange{ - start: 15, - end: 20, - }, - }, - &link{ - start: 35, - end: 47, - text: &bytesRange{ - start: 36, - end: 38, - }, - destination: &bytesRange{ - start: 40, - end: 46, - }, - }, - }, - }, - func(l Link) (Link, error) { - if bytes.Equal(l.GetText(), []byte("b")) { - l.SetDestination([]byte("e.com")) - l.SetText([]byte("d")) - } - if bytes.Equal(l.GetText(), []byte("b1")) { - l.SetDestination([]byte("c2.com")) - } - return l, nil - }, - []byte(`some intro [d](e.com) - some text [b1](c2.com)`), - }, - { - &document{ - []byte("some intro [b](c.com)\n`x`some text [b1](c1.com)\n[b2](c2.com)"), - []Link{ - &link{ - start: 11, - end: 21, - text: &bytesRange{ - start: 12, - end: 13, - }, - destination: &bytesRange{ - start: 15, - end: 20, - }, - }, - &link{ - start: 35, - end: 47, - text: &bytesRange{ - start: 36, - end: 38, - }, - destination: &bytesRange{ - start: 40, - end: 46, - }, - }, - &link{ - start: 48, - end: 60, - text: &bytesRange{ - start: 49, - end: 51, - }, - destination: &bytesRange{ - start: 53, - end: 59, - }, - }, - }, - }, - func(l Link) (Link, error) { - if !bytes.Equal(l.GetText(), []byte("b")) { - l.Remove(false) - return nil, nil - } - return l, nil - }, - []byte("some intro [b](c.com)\n`x`some text \n"), - }, - { - &document{ - []byte("some intro [b](c.com)\n`x`some text [b1](c1.com)\n[b2](c2.com)"), - []Link{ - &link{ - start: 11, - end: 21, - text: &bytesRange{ - start: 12, - end: 13, - }, - destination: &bytesRange{ - start: 15, - end: 20, - }, - }, - &link{ - start: 35, - end: 47, - text: &bytesRange{ - start: 36, - end: 38, - }, - destination: &bytesRange{ - start: 40, - end: 46, - }, - }, - &link{ - start: 48, - end: 60, - text: &bytesRange{ - start: 49, - end: 51, - }, - destination: &bytesRange{ - start: 53, - end: 59, - }, - }, - }, - }, - func(l Link) (Link, error) { - if !bytes.Equal(l.GetText(), []byte("b")) { - l.Remove(true) - return nil, nil - } - return l, nil - }, - []byte("some intro [b](c.com)\n`x`some text b1\nb2"), - }, - } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - for _, l := range tc.in.links { - l.(*link).document = tc.in - } - tc.in.ListLinks(tc.listCb) - got := tc.in.Bytes() - assert.Equal(t, string(tc.want), string(got)) - }) - } -} - -func TestLinkReplaceBytes(t *testing.T) { - testCases := []struct { - in []byte - start int - end int - text []byte - want []byte - }{ - { - []byte("some intro [a](b.com) some text"), - 15, - 20, - []byte("d.com"), - []byte("some intro [a](d.com) some text"), - }, - { - []byte("[a](b.com) some text"), - 1, - 2, - []byte("d"), - []byte("[d](b.com) some text"), - }, - { - []byte("some intro [a](b.com)"), - 12, - 13, - []byte("d"), - []byte("some intro [d](b.com)"), - }, - { - []byte(`Eligendi id aut consequatur sed odit. Aut sit et eaque -ut facilis minima alias dolorum. Provident [explicabo](a.com) -et culpa rerum non soluta.`), - 110, - 115, - []byte("d.com"), - []byte(`Eligendi id aut consequatur sed odit. Aut sit et eaque -ut facilis minima alias dolorum. Provident [explicabo](d.com) -et culpa rerum non soluta.`), - }, - } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - got := replaceBytes(tc.in, tc.start, tc.end, tc.text) - assert.Equal(t, tc.want, got) - }) - } -} - -func TestOffsetLinkByteRange(t *testing.T) { - testCases := []struct { - link *link - offset int - wantLink *link - }{ - { - &link{ - start: 11, - end: 21, - text: &bytesRange{ - start: 12, - end: 13, - }, - destination: &bytesRange{ - start: 15, - end: 20, - }, - }, - 10, - &link{ - start: 21, - end: 31, - text: &bytesRange{ - start: 22, - end: 23, - }, - destination: &bytesRange{ - start: 25, - end: 30, - }, - }, - }, - { - &link{ - start: 11, - end: 21, - text: &bytesRange{ - start: 12, - end: 13, - }, - destination: &bytesRange{ - start: 15, - end: 20, - }, - }, - -10, - &link{ - start: 1, - end: 11, - text: &bytesRange{ - start: 2, - end: 3, - }, - destination: &bytesRange{ - start: 5, - end: 10, - }, - }, - }, - } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - offsetLinkByteRanges(tc.link, tc.offset) - assert.Equal(t, tc.wantLink, tc.link) - }) - } -} diff --git a/pkg/markdown/parser/parser.go b/pkg/markdown/parser/parser.go deleted file mode 100644 index 536e950c..00000000 --- a/pkg/markdown/parser/parser.go +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package parser - -// for each character that triggers a response when parsing inline data. -type inlineParser func(p *parser, data []byte, offset int) (int, Link) - -// Parser embeds properties used by the Parse operation -type parser struct { - insideLink bool - inlineCallback [256]inlineParser - refs map[string]*link -} - -// NewParser creates Parser objects -func NewParser() Parser { - p := parser{} - p.inlineCallback['['] = parseLink - p.inlineCallback['!'] = maybeImage - p.inlineCallback['<'] = parseLeftAngle - p.inlineCallback['h'] = maybeAutoLink - p.inlineCallback['m'] = maybeAutoLink - p.inlineCallback['f'] = maybeAutoLink - p.inlineCallback['H'] = maybeAutoLink - p.inlineCallback['M'] = maybeAutoLink - p.inlineCallback['F'] = maybeAutoLink - p.inlineCallback['`'] = codeSpan - p.refs = make(map[string]*link, 0) - return &p -} - -// Parse scans data and applies callbacks to character patterns -// to model a parsed Document -func (p *parser) Parse(data []byte) Document { - var end int - doc := &document{ - data: data, - links: []Link{}, - } - - n := len(data) - for end < n { - handler := p.inlineCallback[data[end]] - if handler == nil { - end++ - continue - } - consumed, node := handler(p, data, end) - if consumed == 0 { - // no action from the callback - end++ - continue - } - end += consumed - if node != nil { - switch v := node.(type) { - case *link: - { - v.document = doc - doc.links = append(doc.links, node) - } - } - } - } - return doc -} diff --git a/pkg/markdown/parser/parser_test.go b/pkg/markdown/parser/parser_test.go deleted file mode 100644 index 648f5c35..00000000 --- a/pkg/markdown/parser/parser_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package parser - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestParse(t *testing.T) { - testCases := []struct { - in []byte - want *document - }{ - { - []byte("some intro [b](c.com) some text [b1](c1.com)"), - &document{ - links: []Link{ - &link{ - start: 11, - end: 21, - text: &bytesRange{ - start: 12, - end: 13, - }, - destination: &bytesRange{ - start: 15, - end: 20, - }, - }, - &link{ - start: 32, - end: 44, - text: &bytesRange{ - start: 33, - end: 35, - }, - destination: &bytesRange{ - start: 37, - end: 43, - }, - }, - }, - }, - }, - { - []byte(`some intro [b](c.com) - some text [b1](c1.com)`), - &document{ - links: []Link{ - &link{ - start: 11, - end: 21, - text: &bytesRange{ - start: 12, - end: 13, - }, - destination: &bytesRange{ - start: 15, - end: 20, - }, - }, - &link{ - start: 36, - end: 48, - text: &bytesRange{ - start: 37, - end: 39, - }, - destination: &bytesRange{ - start: 41, - end: 47, - }, - }, - }, - }, - }, - } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - tc.want.data = tc.in - for _, l := range tc.want.links { - l.(*link).document = tc.want - } - p := NewParser() - got := p.Parse(tc.in) - assert.Equal(t, tc.want, got) - }) - } -} diff --git a/pkg/markdown/parser/types.go b/pkg/markdown/parser/types.go deleted file mode 100644 index c25639c6..00000000 --- a/pkg/markdown/parser/types.go +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package parser - -// Parser is the interface for parsing data into Document -type Parser interface { - Parse(data []byte) Document -} - -// OnLinkListed is a callback function invoked by the -// ListLinks iterator in Document -type OnLinkListed func(link Link) (Link, error) - -// Document is the markdown model parsed from bytes data -type Document interface { - // ListLinks iterates parsed links in this document - // and invokes cb on every link. Should the link be - // removed, which is signaled by OnLinkListed returning - // nil, ListLinks will update the document links list - // referenced by each link. - ListLinks(cb OnLinkListed) error - // Bytes returns the parsed document content bytes - Bytes() []byte -} - -// Link is a markdown link which can be a normal link or an -// embedded image reference -type Link interface { - SetText(text []byte) - SetDestination(text []byte) - SetTitle(text []byte) - GetText() []byte - GetDestination() []byte - GetTitle() []byte - Remove(leaveText bool) - IsImage() bool - IsAutoLink() bool - IsNormalLink() bool -} diff --git a/pkg/markdown/parser_test.go b/pkg/markdown/parser_test.go new file mode 100644 index 00000000..4745220b --- /dev/null +++ b/pkg/markdown/parser_test.go @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package markdown_test + +import ( + "bytes" + "github.com/gardener/docforge/pkg/markdown" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/yuin/goldmark/ast" +) + +var _ = Describe("Parser", func() { + var ( + md string + doc ast.Node + err error + ) + BeforeEach(func() { + md = "---\ntitle: test\n---\n\n## Heading level 2\n\nI really like using Markdown.\n" + }) + JustBeforeEach(func() { + doc, err = markdown.Parse([]byte(md)) + }) + When("Parse markdown", func() { + It("parse the markdown successfully", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(doc).NotTo(BeNil()) + d, ok := doc.(*ast.Document) + Expect(ok).To(BeTrue()) + Expect(d.Meta()).Should(HaveKeyWithValue("title", "test")) + }) + Context("frontmatter invalid", func() { + BeforeEach(func() { + md = "---\na = b\n---\n\n## Heading level 2\n\nI really like using Markdown.\n" + }) + It("should fails", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("a = b")) + }) + }) + Context("add frontmatter", func() { + var ( + buf *bytes.Buffer + m map[string]interface{} + ) + BeforeEach(func() { + md = "## Heading level 2\n\nI really like using Markdown.\n" + m = make(map[string]interface{}) + m["title"] = "test" + buf = &bytes.Buffer{} + }) + JustBeforeEach(func() { + Expect(doc).NotTo(BeNil()) + d, ok := doc.(*ast.Document) + Expect(ok).To(BeTrue()) + d.SetMeta(m) + rnd := markdown.NewLinkModifierRenderer() + Expect(rnd.Render(buf, []byte(md), d)).To(Succeed()) + }) + It("adds provided frontmatter", func() { + Expect(string(buf.Bytes())).To(Equal("---\ntitle: test\n---\n\n## Heading level 2\n\nI really like using Markdown.\n")) + }) + }) + }) +}) diff --git a/pkg/markdown/spec/commonmark_v.0.30.json b/pkg/markdown/spec/commonmark_v.0.30.json new file mode 100644 index 00000000..d742f941 --- /dev/null +++ b/pkg/markdown/spec/commonmark_v.0.30.json @@ -0,0 +1,5218 @@ +[ + { + "markdown": "\tfoo\tbaz\t\tbim\n", + "html": "
foo\tbaz\t\tbim\n
\n", + "example": 1, + "start_line": 356, + "end_line": 361, + "section": "Tabs" + }, + { + "markdown": " \tfoo\tbaz\t\tbim\n", + "html": "
foo\tbaz\t\tbim\n
\n", + "example": 2, + "start_line": 363, + "end_line": 368, + "section": "Tabs" + }, + { + "markdown": " a\ta\n ὐ\ta\n", + "html": "
a\ta\nὐ\ta\n
\n", + "example": 3, + "start_line": 370, + "end_line": 377, + "section": "Tabs" + }, + { + "markdown": " - foo\n\n\tbar\n", + "html": "
    \n
  • \n

    foo

    \n

    bar

    \n
    • \n
\n", + "example": 4, + "start_line": 383, + "end_line": 394, + "section": "Tabs" + }, + { + "markdown": "- foo\n\n\t\tbar\n", + "html": "
    \n
  • \n

    foo

    \n
      bar\n
    \n
    • \n
\n", + "example": 5, + "start_line": 396, + "end_line": 408, + "section": "Tabs" + }, + { + "markdown": ">\t\tfoo\n", + "html": "
\n
  foo\n
\n
\n", + "example": 6, + "start_line": 419, + "end_line": 426, + "section": "Tabs" + }, + { + "markdown": "-\t\tfoo\n", + "html": "
    \n
  • \n
      foo\n
    \n
    • \n
\n", + "example": 7, + "start_line": 428, + "end_line": 437, + "section": "Tabs" + }, + { + "markdown": " foo\n\tbar\n", + "html": "
foo\nbar\n
\n", + "example": 8, + "start_line": 440, + "end_line": 447, + "section": "Tabs" + }, + { + "markdown": " - foo\n - bar\n\t - baz\n", + "html": "
    \n
  • foo\n
      \n
    • bar\n
        \n
      • baz
        • \n
      \n
      • \n
    \n
    • \n
\n", + "example": 9, + "start_line": 449, + "end_line": 465, + "section": "Tabs" + }, + { + "markdown": "#\tFoo\n", + "html": "

Foo

\n", + "example": 10, + "start_line": 467, + "end_line": 471, + "section": "Tabs" + }, + { + "markdown": "*\t*\t*\t\n", + "html": "
\n", + "example": 11, + "start_line": 473, + "end_line": 477, + "section": "Tabs" + }, + { + "markdown": "\\!\\\"\\#\\$\\%\\&\\'\\(\\)\\*\\+\\,\\-\\.\\/\\:\\;\\<\\=\\>\\?\\@\\[\\\\\\]\\^\\_\\`\\{\\|\\}\\~\n", + "html": "

!"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~

\n", + "example": 12, + "start_line": 490, + "end_line": 494, + "section": "Backslash escapes" + }, + { + "markdown": "\\\t\\A\\a\\ \\3\\φ\\«\n", + "html": "

\\\t\\A\\a\\ \\3\\φ\\«

\n", + "example": 13, + "start_line": 500, + "end_line": 504, + "section": "Backslash escapes" + }, + { + "markdown": "\\*not emphasized*\n\\
not a tag\n\\[not a link](/foo)\n\\`not code`\n1\\. not a list\n\\* not a list\n\\# not a heading\n\\[foo]: /url \"not a reference\"\n\\ö not a character entity\n", + "html": "

*not emphasized*\n<br/> not a tag\n[not a link](/foo)\n`not code`\n1. not a list\n* not a list\n# not a heading\n[foo]: /url "not a reference"\n&ouml; not a character entity

\n", + "example": 14, + "start_line": 510, + "end_line": 530, + "section": "Backslash escapes" + }, + { + "markdown": "\\\\*emphasis*\n", + "html": "

\\emphasis

\n", + "example": 15, + "start_line": 535, + "end_line": 539, + "section": "Backslash escapes" + }, + { + "markdown": "foo\\\nbar\n", + "html": "

foo
\nbar

\n", + "example": 16, + "start_line": 544, + "end_line": 550, + "section": "Backslash escapes" + }, + { + "markdown": "`` \\[\\` ``\n", + "html": "

\\[\\`

\n", + "example": 17, + "start_line": 556, + "end_line": 560, + "section": "Backslash escapes" + }, + { + "markdown": " \\[\\]\n", + "html": "
\\[\\]\n
\n", + "example": 18, + "start_line": 563, + "end_line": 568, + "section": "Backslash escapes" + }, + { + "markdown": "~~~\n\\[\\]\n~~~\n", + "html": "
\\[\\]\n
\n", + "example": 19, + "start_line": 571, + "end_line": 578, + "section": "Backslash escapes" + }, + { + "markdown": "\n", + "html": "

http://example.com?find=\\*

\n", + "example": 20, + "start_line": 581, + "end_line": 585, + "section": "Backslash escapes" + }, + { + "markdown": "\n", + "html": "\n", + "example": 21, + "start_line": 588, + "end_line": 592, + "section": "Backslash escapes" + }, + { + "markdown": "[foo](/bar\\* \"ti\\*tle\")\n", + "html": "

foo

\n", + "example": 22, + "start_line": 598, + "end_line": 602, + "section": "Backslash escapes" + }, + { + "markdown": "[foo]\n\n[foo]: /bar\\* \"ti\\*tle\"\n", + "html": "

foo

\n", + "example": 23, + "start_line": 605, + "end_line": 611, + "section": "Backslash escapes" + }, + { + "markdown": "``` foo\\+bar\nfoo\n```\n", + "html": "
foo\n
\n", + "example": 24, + "start_line": 614, + "end_line": 621, + "section": "Backslash escapes" + }, + { + "markdown": "  & © Æ Ď\n¾ ℋ ⅆ\n∲ ≧̸\n", + "html": "

  & © Æ Ď\n¾ ℋ ⅆ\n∲ ≧̸

\n", + "example": 25, + "start_line": 650, + "end_line": 658, + "section": "Entity and numeric character references" + }, + { + "markdown": "# Ӓ Ϡ �\n", + "html": "

# Ӓ Ϡ �

\n", + "example": 26, + "start_line": 669, + "end_line": 673, + "section": "Entity and numeric character references" + }, + { + "markdown": "" ആ ಫ\n", + "html": "

" ആ ಫ

\n", + "example": 27, + "start_line": 682, + "end_line": 686, + "section": "Entity and numeric character references" + }, + { + "markdown": "  &x; &#; &#x;\n�\n&#abcdef0;\n&ThisIsNotDefined; &hi?;\n", + "html": "

&nbsp &x; &#; &#x;\n&#87654321;\n&#abcdef0;\n&ThisIsNotDefined; &hi?;

\n", + "example": 28, + "start_line": 691, + "end_line": 701, + "section": "Entity and numeric character references" + }, + { + "markdown": "©\n", + "html": "

&copy

\n", + "example": 29, + "start_line": 708, + "end_line": 712, + "section": "Entity and numeric character references" + }, + { + "markdown": "&MadeUpEntity;\n", + "html": "

&MadeUpEntity;

\n", + "example": 30, + "start_line": 718, + "end_line": 722, + "section": "Entity and numeric character references" + }, + { + "markdown": "\n", + "html": "\n", + "example": 31, + "start_line": 729, + "end_line": 733, + "section": "Entity and numeric character references" + }, + { + "markdown": "[foo](/föö \"föö\")\n", + "html": "

foo

\n", + "example": 32, + "start_line": 736, + "end_line": 740, + "section": "Entity and numeric character references" + }, + { + "markdown": "[foo]\n\n[foo]: /föö \"föö\"\n", + "html": "

foo

\n", + "example": 33, + "start_line": 743, + "end_line": 749, + "section": "Entity and numeric character references" + }, + { + "markdown": "``` föö\nfoo\n```\n", + "html": "
foo\n
\n", + "example": 34, + "start_line": 752, + "end_line": 759, + "section": "Entity and numeric character references" + }, + { + "markdown": "`föö`\n", + "html": "

f&ouml;&ouml;

\n", + "example": 35, + "start_line": 765, + "end_line": 769, + "section": "Entity and numeric character references" + }, + { + "markdown": " föfö\n", + "html": "
f&ouml;f&ouml;\n
\n", + "example": 36, + "start_line": 772, + "end_line": 777, + "section": "Entity and numeric character references" + }, + { + "markdown": "*foo*\n*foo*\n", + "html": "

*foo*\nfoo

\n", + "example": 37, + "start_line": 784, + "end_line": 790, + "section": "Entity and numeric character references" + }, + { + "markdown": "* foo\n\n* foo\n", + "html": "

* foo

\n
    \n
  • foo
    • \n
\n", + "example": 38, + "start_line": 792, + "end_line": 801, + "section": "Entity and numeric character references" + }, + { + "markdown": "foo bar\n", + "html": "

foo\n\nbar

\n", + "example": 39, + "start_line": 803, + "end_line": 809, + "section": "Entity and numeric character references" + }, + { + "markdown": " foo\n", + "html": "

\tfoo

\n", + "example": 40, + "start_line": 811, + "end_line": 815, + "section": "Entity and numeric character references" + }, + { + "markdown": "[a](url "tit")\n", + "html": "

[a](url "tit")

\n", + "example": 41, + "start_line": 818, + "end_line": 822, + "section": "Entity and numeric character references" + }, + { + "markdown": "- `one\n- two`\n", + "html": "
    \n
  • `one
    • \n
  • two`
    • \n
\n", + "example": 42, + "start_line": 841, + "end_line": 849, + "section": "Precedence" + }, + { + "markdown": "***\n---\n___\n", + "html": "
\n
\n
\n", + "example": 43, + "start_line": 880, + "end_line": 888, + "section": "Thematic breaks" + }, + { + "markdown": "+++\n", + "html": "

+++

\n", + "example": 44, + "start_line": 893, + "end_line": 897, + "section": "Thematic breaks" + }, + { + "markdown": "===\n", + "html": "

===

\n", + "example": 45, + "start_line": 900, + "end_line": 904, + "section": "Thematic breaks" + }, + { + "markdown": "--\n**\n__\n", + "html": "

--\n**\n__

\n", + "example": 46, + "start_line": 909, + "end_line": 917, + "section": "Thematic breaks" + }, + { + "markdown": " ***\n ***\n ***\n", + "html": "
\n
\n
\n", + "example": 47, + "start_line": 922, + "end_line": 930, + "section": "Thematic breaks" + }, + { + "markdown": " ***\n", + "html": "
***\n
\n", + "example": 48, + "start_line": 935, + "end_line": 940, + "section": "Thematic breaks" + }, + { + "markdown": "Foo\n ***\n", + "html": "

Foo\n***

\n", + "example": 49, + "start_line": 943, + "end_line": 949, + "section": "Thematic breaks" + }, + { + "markdown": "_____________________________________\n", + "html": "
\n", + "example": 50, + "start_line": 954, + "end_line": 958, + "section": "Thematic breaks" + }, + { + "markdown": " - - -\n", + "html": "
\n", + "example": 51, + "start_line": 963, + "end_line": 967, + "section": "Thematic breaks" + }, + { + "markdown": " ** * ** * ** * **\n", + "html": "
\n", + "example": 52, + "start_line": 970, + "end_line": 974, + "section": "Thematic breaks" + }, + { + "markdown": "- - - -\n", + "html": "
\n", + "example": 53, + "start_line": 977, + "end_line": 981, + "section": "Thematic breaks" + }, + { + "markdown": "- - - - \n", + "html": "
\n", + "example": 54, + "start_line": 986, + "end_line": 990, + "section": "Thematic breaks" + }, + { + "markdown": "_ _ _ _ a\n\na------\n\n---a---\n", + "html": "

_ _ _ _ a

\n

a------

\n

---a---

\n", + "example": 55, + "start_line": 995, + "end_line": 1005, + "section": "Thematic breaks" + }, + { + "markdown": " *-*\n", + "html": "

-

\n", + "example": 56, + "start_line": 1011, + "end_line": 1015, + "section": "Thematic breaks" + }, + { + "markdown": "- foo\n***\n- bar\n", + "html": "
    \n
  • foo
    • \n
\n
\n
    \n
  • bar
    • \n
\n", + "example": 57, + "start_line": 1020, + "end_line": 1032, + "section": "Thematic breaks" + }, + { + "markdown": "Foo\n***\nbar\n", + "html": "

Foo

\n
\n

bar

\n", + "example": 58, + "start_line": 1037, + "end_line": 1045, + "section": "Thematic breaks" + }, + { + "markdown": "Foo\n---\nbar\n", + "html": "

Foo

\n

bar

\n", + "example": 59, + "start_line": 1054, + "end_line": 1061, + "section": "Thematic breaks" + }, + { + "markdown": "* Foo\n* * *\n* Bar\n", + "html": "
    \n
  • Foo
    • \n
\n
\n
    \n
  • Bar
    • \n
\n", + "example": 60, + "start_line": 1067, + "end_line": 1079, + "section": "Thematic breaks" + }, + { + "markdown": "- Foo\n- * * *\n", + "html": "
    \n
  • Foo
    • \n
  • \n
    \n
    • \n
\n", + "example": 61, + "start_line": 1084, + "end_line": 1094, + "section": "Thematic breaks" + }, + { + "markdown": "# foo\n## foo\n### foo\n#### foo\n##### foo\n###### foo\n", + "html": "

foo

\n

foo

\n

foo

\n

foo

\n
foo
\n
foo
\n", + "example": 62, + "start_line": 1113, + "end_line": 1127, + "section": "ATX headings" + }, + { + "markdown": "####### foo\n", + "html": "

####### foo

\n", + "example": 63, + "start_line": 1132, + "end_line": 1136, + "section": "ATX headings" + }, + { + "markdown": "#5 bolt\n\n#hashtag\n", + "html": "

#5 bolt

\n

#hashtag

\n", + "example": 64, + "start_line": 1147, + "end_line": 1154, + "section": "ATX headings" + }, + { + "markdown": "\\## foo\n", + "html": "

## foo

\n", + "example": 65, + "start_line": 1159, + "end_line": 1163, + "section": "ATX headings" + }, + { + "markdown": "# foo *bar* \\*baz\\*\n", + "html": "

foo bar *baz*

\n", + "example": 66, + "start_line": 1168, + "end_line": 1172, + "section": "ATX headings" + }, + { + "markdown": "# foo \n", + "html": "

foo

\n", + "example": 67, + "start_line": 1177, + "end_line": 1181, + "section": "ATX headings" + }, + { + "markdown": " ### foo\n ## foo\n # foo\n", + "html": "

foo

\n

foo

\n

foo

\n", + "example": 68, + "start_line": 1186, + "end_line": 1194, + "section": "ATX headings" + }, + { + "markdown": " # foo\n", + "html": "
# foo\n
\n", + "example": 69, + "start_line": 1199, + "end_line": 1204, + "section": "ATX headings" + }, + { + "markdown": "foo\n # bar\n", + "html": "

foo\n# bar

\n", + "example": 70, + "start_line": 1207, + "end_line": 1213, + "section": "ATX headings" + }, + { + "markdown": "## foo ##\n ### bar ###\n", + "html": "

foo

\n

bar

\n", + "example": 71, + "start_line": 1218, + "end_line": 1224, + "section": "ATX headings" + }, + { + "markdown": "# foo ##################################\n##### foo ##\n", + "html": "

foo

\n
foo
\n", + "example": 72, + "start_line": 1229, + "end_line": 1235, + "section": "ATX headings" + }, + { + "markdown": "### foo ### \n", + "html": "

foo

\n", + "example": 73, + "start_line": 1240, + "end_line": 1244, + "section": "ATX headings" + }, + { + "markdown": "### foo ### b\n", + "html": "

foo ### b

\n", + "example": 74, + "start_line": 1251, + "end_line": 1255, + "section": "ATX headings" + }, + { + "markdown": "# foo#\n", + "html": "

foo#

\n", + "example": 75, + "start_line": 1260, + "end_line": 1264, + "section": "ATX headings" + }, + { + "markdown": "### foo \\###\n## foo #\\##\n# foo \\#\n", + "html": "

foo ###

\n

foo ###

\n

foo #

\n", + "example": 76, + "start_line": 1270, + "end_line": 1278, + "section": "ATX headings" + }, + { + "markdown": "****\n## foo\n****\n", + "html": "
\n

foo

\n
\n", + "example": 77, + "start_line": 1284, + "end_line": 1292, + "section": "ATX headings" + }, + { + "markdown": "Foo bar\n# baz\nBar foo\n", + "html": "

Foo bar

\n

baz

\n

Bar foo

\n", + "example": 78, + "start_line": 1295, + "end_line": 1303, + "section": "ATX headings" + }, + { + "markdown": "## \n#\n### ###\n", + "html": "

\n

\n

\n", + "example": 79, + "start_line": 1308, + "end_line": 1316, + "section": "ATX headings" + }, + { + "markdown": "Foo *bar*\n=========\n\nFoo *bar*\n---------\n", + "html": "

Foo bar

\n

Foo bar

\n", + "example": 80, + "start_line": 1351, + "end_line": 1360, + "section": "Setext headings" + }, + { + "markdown": "Foo *bar\nbaz*\n====\n", + "html": "

Foo bar\nbaz

\n", + "example": 81, + "start_line": 1365, + "end_line": 1372, + "section": "Setext headings" + }, + { + "markdown": " Foo *bar\nbaz*\t\n====\n", + "html": "

Foo bar\nbaz

\n", + "example": 82, + "start_line": 1379, + "end_line": 1386, + "section": "Setext headings" + }, + { + "markdown": "Foo\n-------------------------\n\nFoo\n=\n", + "html": "

Foo

\n

Foo

\n", + "example": 83, + "start_line": 1391, + "end_line": 1400, + "section": "Setext headings" + }, + { + "markdown": " Foo\n---\n\n Foo\n-----\n\n Foo\n ===\n", + "html": "

Foo

\n

Foo

\n

Foo

\n", + "example": 84, + "start_line": 1406, + "end_line": 1419, + "section": "Setext headings" + }, + { + "markdown": " Foo\n ---\n\n Foo\n---\n", + "html": "
Foo\n---\n\nFoo\n
\n
\n", + "example": 85, + "start_line": 1424, + "end_line": 1437, + "section": "Setext headings" + }, + { + "markdown": "Foo\n ---- \n", + "html": "

Foo

\n", + "example": 86, + "start_line": 1443, + "end_line": 1448, + "section": "Setext headings" + }, + { + "markdown": "Foo\n ---\n", + "html": "

Foo\n---

\n", + "example": 87, + "start_line": 1453, + "end_line": 1459, + "section": "Setext headings" + }, + { + "markdown": "Foo\n= =\n\nFoo\n--- -\n", + "html": "

Foo\n= =

\n

Foo

\n
\n", + "example": 88, + "start_line": 1464, + "end_line": 1475, + "section": "Setext headings" + }, + { + "markdown": "Foo \n-----\n", + "html": "

Foo

\n", + "example": 89, + "start_line": 1480, + "end_line": 1485, + "section": "Setext headings" + }, + { + "markdown": "Foo\\\n----\n", + "html": "

Foo\\

\n", + "example": 90, + "start_line": 1490, + "end_line": 1495, + "section": "Setext headings" + }, + { + "markdown": "`Foo\n----\n`\n\n\n", + "html": "

`Foo

\n

`

\n

<a title="a lot

\n

of dashes"/>

\n", + "example": 91, + "start_line": 1501, + "end_line": 1514, + "section": "Setext headings" + }, + { + "markdown": "> Foo\n---\n", + "html": "
\n

Foo

\n
\n
\n", + "example": 92, + "start_line": 1520, + "end_line": 1528, + "section": "Setext headings" + }, + { + "markdown": "> foo\nbar\n===\n", + "html": "
\n

foo\nbar\n===

\n
\n", + "example": 93, + "start_line": 1531, + "end_line": 1541, + "section": "Setext headings" + }, + { + "markdown": "- Foo\n---\n", + "html": "
    \n
  • Foo
    • \n
\n
\n", + "example": 94, + "start_line": 1544, + "end_line": 1552, + "section": "Setext headings" + }, + { + "markdown": "Foo\nBar\n---\n", + "html": "

Foo\nBar

\n", + "example": 95, + "start_line": 1559, + "end_line": 1566, + "section": "Setext headings" + }, + { + "markdown": "---\nFoo\n---\nBar\n---\nBaz\n", + "html": "
\n

Foo

\n

Bar

\n

Baz

\n", + "example": 96, + "start_line": 1572, + "end_line": 1584, + "section": "Setext headings" + }, + { + "markdown": "\n====\n", + "html": "

====

\n", + "example": 97, + "start_line": 1589, + "end_line": 1594, + "section": "Setext headings" + }, + { + "markdown": "---\n---\n", + "html": "
\n
\n", + "example": 98, + "start_line": 1601, + "end_line": 1607, + "section": "Setext headings" + }, + { + "markdown": "- foo\n-----\n", + "html": "
    \n
  • foo
    • \n
\n
\n", + "example": 99, + "start_line": 1610, + "end_line": 1618, + "section": "Setext headings" + }, + { + "markdown": " foo\n---\n", + "html": "
foo\n
\n
\n", + "example": 100, + "start_line": 1621, + "end_line": 1628, + "section": "Setext headings" + }, + { + "markdown": "> foo\n-----\n", + "html": "
\n

foo

\n
\n
\n", + "example": 101, + "start_line": 1631, + "end_line": 1639, + "section": "Setext headings" + }, + { + "markdown": "\\> foo\n------\n", + "html": "

> foo

\n", + "example": 102, + "start_line": 1645, + "end_line": 1650, + "section": "Setext headings" + }, + { + "markdown": "Foo\n\nbar\n---\nbaz\n", + "html": "

Foo

\n

bar

\n

baz

\n", + "example": 103, + "start_line": 1676, + "end_line": 1686, + "section": "Setext headings" + }, + { + "markdown": "Foo\nbar\n\n---\n\nbaz\n", + "html": "

Foo\nbar

\n
\n

baz

\n", + "example": 104, + "start_line": 1692, + "end_line": 1704, + "section": "Setext headings" + }, + { + "markdown": "Foo\nbar\n* * *\nbaz\n", + "html": "

Foo\nbar

\n
\n

baz

\n", + "example": 105, + "start_line": 1710, + "end_line": 1720, + "section": "Setext headings" + }, + { + "markdown": "Foo\nbar\n\\---\nbaz\n", + "html": "

Foo\nbar\n---\nbaz

\n", + "example": 106, + "start_line": 1725, + "end_line": 1735, + "section": "Setext headings" + }, + { + "markdown": " a simple\n indented code block\n", + "html": "
a simple\n  indented code block\n
\n", + "example": 107, + "start_line": 1753, + "end_line": 1760, + "section": "Indented code blocks" + }, + { + "markdown": " - foo\n\n bar\n", + "html": "
    \n
  • \n

    foo

    \n

    bar

    \n
    • \n
\n", + "example": 108, + "start_line": 1767, + "end_line": 1778, + "section": "Indented code blocks" + }, + { + "markdown": "1. foo\n\n - bar\n", + "html": "
    \n
  1. \n

    foo

    \n
      \n
    • bar
      • \n
    \n
    2. \n
\n", + "example": 109, + "start_line": 1781, + "end_line": 1794, + "section": "Indented code blocks" + }, + { + "markdown": " \n *hi*\n\n - one\n", + "html": "
<a/>\n*hi*\n\n- one\n
\n", + "example": 110, + "start_line": 1801, + "end_line": 1812, + "section": "Indented code blocks" + }, + { + "markdown": " chunk1\n\n chunk2\n \n \n \n chunk3\n", + "html": "
chunk1\n\nchunk2\n\n\n\nchunk3\n
\n", + "example": 111, + "start_line": 1817, + "end_line": 1834, + "section": "Indented code blocks" + }, + { + "markdown": " chunk1\n \n chunk2\n", + "html": "
chunk1\n  \n  chunk2\n
\n", + "example": 112, + "start_line": 1840, + "end_line": 1849, + "section": "Indented code blocks" + }, + { + "markdown": "Foo\n bar\n\n", + "html": "

Foo\nbar

\n", + "example": 113, + "start_line": 1855, + "end_line": 1862, + "section": "Indented code blocks" + }, + { + "markdown": " foo\nbar\n", + "html": "
foo\n
\n

bar

\n", + "example": 114, + "start_line": 1869, + "end_line": 1876, + "section": "Indented code blocks" + }, + { + "markdown": "# Heading\n foo\nHeading\n------\n foo\n----\n", + "html": "

Heading

\n
foo\n
\n

Heading

\n
foo\n
\n
\n", + "example": 115, + "start_line": 1882, + "end_line": 1897, + "section": "Indented code blocks" + }, + { + "markdown": " foo\n bar\n", + "html": "
    foo\nbar\n
\n", + "example": 116, + "start_line": 1902, + "end_line": 1909, + "section": "Indented code blocks" + }, + { + "markdown": "\n \n foo\n \n\n", + "html": "
foo\n
\n", + "example": 117, + "start_line": 1915, + "end_line": 1924, + "section": "Indented code blocks" + }, + { + "markdown": " foo \n", + "html": "
foo  \n
\n", + "example": 118, + "start_line": 1929, + "end_line": 1934, + "section": "Indented code blocks" + }, + { + "markdown": "```\n<\n >\n```\n", + "html": "
<\n >\n
\n", + "example": 119, + "start_line": 1984, + "end_line": 1993, + "section": "Fenced code blocks" + }, + { + "markdown": "~~~\n<\n >\n~~~\n", + "html": "
<\n >\n
\n", + "example": 120, + "start_line": 1998, + "end_line": 2007, + "section": "Fenced code blocks" + }, + { + "markdown": "``\nfoo\n``\n", + "html": "

foo

\n", + "example": 121, + "start_line": 2011, + "end_line": 2017, + "section": "Fenced code blocks" + }, + { + "markdown": "```\naaa\n~~~\n```\n", + "html": "
aaa\n~~~\n
\n", + "example": 122, + "start_line": 2022, + "end_line": 2031, + "section": "Fenced code blocks" + }, + { + "markdown": "~~~\naaa\n```\n~~~\n", + "html": "
aaa\n```\n
\n", + "example": 123, + "start_line": 2034, + "end_line": 2043, + "section": "Fenced code blocks" + }, + { + "markdown": "````\naaa\n```\n``````\n", + "html": "
aaa\n```\n
\n", + "example": 124, + "start_line": 2048, + "end_line": 2057, + "section": "Fenced code blocks" + }, + { + "markdown": "~~~~\naaa\n~~~\n~~~~\n", + "html": "
aaa\n~~~\n
\n", + "example": 125, + "start_line": 2060, + "end_line": 2069, + "section": "Fenced code blocks" + }, + { + "markdown": "```\n", + "html": "
\n", + "example": 126, + "start_line": 2075, + "end_line": 2079, + "section": "Fenced code blocks" + }, + { + "markdown": "`````\n\n```\naaa\n", + "html": "
\n```\naaa\n
\n", + "example": 127, + "start_line": 2082, + "end_line": 2092, + "section": "Fenced code blocks" + }, + { + "markdown": "> ```\n> aaa\n\nbbb\n", + "html": "
\n
aaa\n
\n
\n

bbb

\n", + "example": 128, + "start_line": 2095, + "end_line": 2106, + "section": "Fenced code blocks" + }, + { + "markdown": "```\n\n \n```\n", + "html": "
\n  \n
\n", + "example": 129, + "start_line": 2111, + "end_line": 2120, + "section": "Fenced code blocks" + }, + { + "markdown": "```\n```\n", + "html": "
\n", + "example": 130, + "start_line": 2125, + "end_line": 2130, + "section": "Fenced code blocks" + }, + { + "markdown": " ```\n aaa\naaa\n```\n", + "html": "
aaa\naaa\n
\n", + "example": 131, + "start_line": 2137, + "end_line": 2146, + "section": "Fenced code blocks" + }, + { + "markdown": " ```\naaa\n aaa\naaa\n ```\n", + "html": "
aaa\naaa\naaa\n
\n", + "example": 132, + "start_line": 2149, + "end_line": 2160, + "section": "Fenced code blocks" + }, + { + "markdown": " ```\n aaa\n aaa\n aaa\n ```\n", + "html": "
aaa\n aaa\naaa\n
\n", + "example": 133, + "start_line": 2163, + "end_line": 2174, + "section": "Fenced code blocks" + }, + { + "markdown": " ```\n aaa\n ```\n", + "html": "
```\naaa\n```\n
\n", + "example": 134, + "start_line": 2179, + "end_line": 2188, + "section": "Fenced code blocks" + }, + { + "markdown": "```\naaa\n ```\n", + "html": "
aaa\n
\n", + "example": 135, + "start_line": 2194, + "end_line": 2201, + "section": "Fenced code blocks" + }, + { + "markdown": " ```\naaa\n ```\n", + "html": "
aaa\n
\n", + "example": 136, + "start_line": 2204, + "end_line": 2211, + "section": "Fenced code blocks" + }, + { + "markdown": "```\naaa\n ```\n", + "html": "
aaa\n    ```\n
\n", + "example": 137, + "start_line": 2216, + "end_line": 2224, + "section": "Fenced code blocks" + }, + { + "markdown": "``` ```\naaa\n", + "html": "

\naaa

\n", + "example": 138, + "start_line": 2230, + "end_line": 2236, + "section": "Fenced code blocks" + }, + { + "markdown": "~~~~~~\naaa\n~~~ ~~\n", + "html": "
aaa\n~~~ ~~\n
\n", + "example": 139, + "start_line": 2239, + "end_line": 2247, + "section": "Fenced code blocks" + }, + { + "markdown": "foo\n```\nbar\n```\nbaz\n", + "html": "

foo

\n
bar\n
\n

baz

\n", + "example": 140, + "start_line": 2253, + "end_line": 2264, + "section": "Fenced code blocks" + }, + { + "markdown": "foo\n---\n~~~\nbar\n~~~\n# baz\n", + "html": "

foo

\n
bar\n
\n

baz

\n", + "example": 141, + "start_line": 2270, + "end_line": 2282, + "section": "Fenced code blocks" + }, + { + "markdown": "```ruby\ndef foo(x)\n return 3\nend\n```\n", + "html": "
def foo(x)\n  return 3\nend\n
\n", + "example": 142, + "start_line": 2292, + "end_line": 2303, + "section": "Fenced code blocks" + }, + { + "markdown": "~~~~ ruby startline=3 $%@#$\ndef foo(x)\n return 3\nend\n~~~~~~~\n", + "html": "
def foo(x)\n  return 3\nend\n
\n", + "example": 143, + "start_line": 2306, + "end_line": 2317, + "section": "Fenced code blocks" + }, + { + "markdown": "````;\n````\n", + "html": "
\n", + "example": 144, + "start_line": 2320, + "end_line": 2325, + "section": "Fenced code blocks" + }, + { + "markdown": "``` aa ```\nfoo\n", + "html": "

aa\nfoo

\n", + "example": 145, + "start_line": 2330, + "end_line": 2336, + "section": "Fenced code blocks" + }, + { + "markdown": "~~~ aa ``` ~~~\nfoo\n~~~\n", + "html": "
foo\n
\n", + "example": 146, + "start_line": 2341, + "end_line": 2348, + "section": "Fenced code blocks" + }, + { + "markdown": "```\n``` aaa\n```\n", + "html": "
``` aaa\n
\n", + "example": 147, + "start_line": 2353, + "end_line": 2360, + "section": "Fenced code blocks" + }, + { + "markdown": "
\n
\n**Hello**,\n\n_world_.\n
\n
\n", + "html": "
\n
\n**Hello**,\n
world.\n

\n
\n", + "example": 148, + "start_line": 2432, + "end_line": 2447, + "section": "HTML blocks" + }, + { + "markdown": "\n \n \n \n
\n hi\n
\n\nokay.\n", + "html": "\n \n \n \n
\n hi\n
\n

okay.

\n", + "example": 149, + "start_line": 2461, + "end_line": 2480, + "section": "HTML blocks" + }, + { + "markdown": " \n*foo*\n", + "example": 151, + "start_line": 2496, + "end_line": 2502, + "section": "HTML blocks" + }, + { + "markdown": "
\n\n*Markdown*\n\n
\n", + "html": "
\n

Markdown

\n
\n", + "example": 152, + "start_line": 2507, + "end_line": 2517, + "section": "HTML blocks" + }, + { + "markdown": "
\n
\n", + "html": "
\n
\n", + "example": 153, + "start_line": 2523, + "end_line": 2531, + "section": "HTML blocks" + }, + { + "markdown": "
\n
\n", + "html": "
\n
\n", + "example": 154, + "start_line": 2534, + "end_line": 2542, + "section": "HTML blocks" + }, + { + "markdown": "
\n*foo*\n\n*bar*\n", + "html": "
\n*foo*\n

bar

\n", + "example": 155, + "start_line": 2546, + "end_line": 2555, + "section": "HTML blocks" + }, + { + "markdown": "\n", + "html": "\n", + "example": 159, + "start_line": 2595, + "end_line": 2599, + "section": "HTML blocks" + }, + { + "markdown": "
\nfoo\n
\n", + "html": "
\nfoo\n
\n", + "example": 160, + "start_line": 2602, + "end_line": 2610, + "section": "HTML blocks" + }, + { + "markdown": "
\n``` c\nint x = 33;\n```\n", + "html": "
\n``` c\nint x = 33;\n```\n", + "example": 161, + "start_line": 2619, + "end_line": 2629, + "section": "HTML blocks" + }, + { + "markdown": "\n*bar*\n\n", + "html": "\n*bar*\n\n", + "example": 162, + "start_line": 2636, + "end_line": 2644, + "section": "HTML blocks" + }, + { + "markdown": "\n*bar*\n\n", + "html": "\n*bar*\n\n", + "example": 163, + "start_line": 2649, + "end_line": 2657, + "section": "HTML blocks" + }, + { + "markdown": "\n*bar*\n\n", + "html": "\n*bar*\n\n", + "example": 164, + "start_line": 2660, + "end_line": 2668, + "section": "HTML blocks" + }, + { + "markdown": "\n*bar*\n", + "html": "\n*bar*\n", + "example": 165, + "start_line": 2671, + "end_line": 2677, + "section": "HTML blocks" + }, + { + "markdown": "\n*foo*\n\n", + "html": "\n*foo*\n\n", + "example": 166, + "start_line": 2686, + "end_line": 2694, + "section": "HTML blocks" + }, + { + "markdown": "\n\n*foo*\n\n\n", + "html": "\n

foo

\n\n", + "example": 167, + "start_line": 2701, + "end_line": 2711, + "section": "HTML blocks" + }, + { + "markdown": "*foo*\n", + "html": "

foo

\n", + "example": 168, + "start_line": 2719, + "end_line": 2723, + "section": "HTML blocks" + }, + { + "markdown": "
\nimport Text.HTML.TagSoup\n\nmain :: IO ()\nmain = print $ parseTags tags\n
\nokay\n", + "html": "
\nimport Text.HTML.TagSoup\n\nmain :: IO ()\nmain = print $ parseTags tags\n
\n

okay

\n", + "example": 169, + "start_line": 2735, + "end_line": 2751, + "section": "HTML blocks" + }, + { + "markdown": "\nokay\n", + "html": "\n

okay

\n", + "example": 170, + "start_line": 2756, + "end_line": 2770, + "section": "HTML blocks" + }, + { + "markdown": "\n", + "html": "\n", + "example": 171, + "start_line": 2775, + "end_line": 2791, + "section": "HTML blocks" + }, + { + "markdown": "\nh1 {color:red;}\n\np {color:blue;}\n\nokay\n", + "html": "\nh1 {color:red;}\n\np {color:blue;}\n\n

okay

\n", + "example": 172, + "start_line": 2795, + "end_line": 2811, + "section": "HTML blocks" + }, + { + "markdown": "\n\nfoo\n", + "html": "\n\nfoo\n", + "example": 173, + "start_line": 2818, + "end_line": 2828, + "section": "HTML blocks" + }, + { + "markdown": ">
\n> foo\n\nbar\n", + "html": "
\n
\nfoo\n
\n

bar

\n", + "example": 174, + "start_line": 2831, + "end_line": 2842, + "section": "HTML blocks" + }, + { + "markdown": "-
\n- foo\n", + "html": "
    \n
  • \n
    \n
    • \n
  • foo
    • \n
\n", + "example": 175, + "start_line": 2845, + "end_line": 2855, + "section": "HTML blocks" + }, + { + "markdown": "\n*foo*\n", + "html": "\n

foo

\n", + "example": 176, + "start_line": 2860, + "end_line": 2866, + "section": "HTML blocks" + }, + { + "markdown": "*bar*\n*baz*\n", + "html": "*bar*\n

baz

\n", + "example": 177, + "start_line": 2869, + "end_line": 2875, + "section": "HTML blocks" + }, + { + "markdown": "1. *bar*\n", + "html": "1. *bar*\n", + "example": 178, + "start_line": 2881, + "end_line": 2889, + "section": "HTML blocks" + }, + { + "markdown": "\nokay\n", + "html": "\n

okay

\n", + "example": 179, + "start_line": 2894, + "end_line": 2906, + "section": "HTML blocks" + }, + { + "markdown": "';\n\n?>\nokay\n", + "html": "';\n\n?>\n

okay

\n", + "example": 180, + "start_line": 2912, + "end_line": 2926, + "section": "HTML blocks" + }, + { + "markdown": "\n", + "html": "\n", + "example": 181, + "start_line": 2931, + "end_line": 2935, + "section": "HTML blocks" + }, + { + "markdown": "\nokay\n", + "html": "\n

okay

\n", + "example": 182, + "start_line": 2940, + "end_line": 2968, + "section": "HTML blocks" + }, + { + "markdown": " \n\n \n", + "html": " \n
<!-- foo -->\n
\n", + "example": 183, + "start_line": 2974, + "end_line": 2982, + "section": "HTML blocks" + }, + { + "markdown": "
\n\n
\n", + "html": "
\n
<div>\n
\n", + "example": 184, + "start_line": 2985, + "end_line": 2993, + "section": "HTML blocks" + }, + { + "markdown": "Foo\n
\nbar\n
\n", + "html": "

Foo

\n
\nbar\n
\n", + "example": 185, + "start_line": 2999, + "end_line": 3009, + "section": "HTML blocks" + }, + { + "markdown": "
\nbar\n
\n*foo*\n", + "html": "
\nbar\n
\n*foo*\n", + "example": 186, + "start_line": 3016, + "end_line": 3026, + "section": "HTML blocks" + }, + { + "markdown": "Foo\n\nbaz\n", + "html": "

Foo\n\nbaz

\n", + "example": 187, + "start_line": 3031, + "end_line": 3039, + "section": "HTML blocks" + }, + { + "markdown": "
\n\n*Emphasized* text.\n\n
\n", + "html": "
\n

Emphasized text.

\n
\n", + "example": 188, + "start_line": 3072, + "end_line": 3082, + "section": "HTML blocks" + }, + { + "markdown": "
\n*Emphasized* text.\n
\n", + "html": "
\n*Emphasized* text.\n
\n", + "example": 189, + "start_line": 3085, + "end_line": 3093, + "section": "HTML blocks" + }, + { + "markdown": "\n\n\n\n\n\n\n\n
\nHi\n
\n", + "html": "\n\n\n\n
\nHi\n
\n", + "example": 190, + "start_line": 3107, + "end_line": 3127, + "section": "HTML blocks" + }, + { + "markdown": "\n\n \n\n \n\n \n\n
\n Hi\n
\n", + "html": "\n \n
<td>\n  Hi\n</td>\n
\n \n
\n", + "example": 191, + "start_line": 3134, + "end_line": 3155, + "section": "HTML blocks" + }, + { + "markdown": "[foo]: /url \"title\"\n\n[foo]\n", + "html": "

foo

\n", + "example": 192, + "start_line": 3183, + "end_line": 3189, + "section": "Link reference definitions" + }, + { + "markdown": " [foo]: \n /url \n 'the title' \n\n[foo]\n", + "html": "

foo

\n", + "example": 193, + "start_line": 3192, + "end_line": 3200, + "section": "Link reference definitions" + }, + { + "markdown": "[Foo*bar\\]]:my_(url) 'title (with parens)'\n\n[Foo*bar\\]]\n", + "html": "

Foo*bar]

\n", + "example": 194, + "start_line": 3203, + "end_line": 3209, + "section": "Link reference definitions" + }, + { + "markdown": "[Foo bar]:\n\n'title'\n\n[Foo bar]\n", + "html": "

Foo bar

\n", + "example": 195, + "start_line": 3212, + "end_line": 3220, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]: /url '\ntitle\nline1\nline2\n'\n\n[foo]\n", + "html": "

foo

\n", + "example": 196, + "start_line": 3225, + "end_line": 3239, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]: /url 'title\n\nwith blank line'\n\n[foo]\n", + "html": "

[foo]: /url 'title

\n

with blank line'

\n

[foo]

\n", + "example": 197, + "start_line": 3244, + "end_line": 3254, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]:\n/url\n\n[foo]\n", + "html": "

foo

\n", + "example": 198, + "start_line": 3259, + "end_line": 3266, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]:\n\n[foo]\n", + "html": "

[foo]:

\n

[foo]

\n", + "example": 199, + "start_line": 3271, + "end_line": 3278, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]: <>\n\n[foo]\n", + "html": "

foo

\n", + "example": 200, + "start_line": 3283, + "end_line": 3289, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]: (baz)\n\n[foo]\n", + "html": "

[foo]: (baz)

\n

[foo]

\n", + "example": 201, + "start_line": 3294, + "end_line": 3301, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]: /url\\bar\\*baz \"foo\\\"bar\\baz\"\n\n[foo]\n", + "html": "

foo

\n", + "example": 202, + "start_line": 3307, + "end_line": 3313, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]\n\n[foo]: url\n", + "html": "

foo

\n", + "example": 203, + "start_line": 3318, + "end_line": 3324, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]\n\n[foo]: first\n[foo]: second\n", + "html": "

foo

\n", + "example": 204, + "start_line": 3330, + "end_line": 3337, + "section": "Link reference definitions" + }, + { + "markdown": "[FOO]: /url\n\n[Foo]\n", + "html": "

Foo

\n", + "example": 205, + "start_line": 3343, + "end_line": 3349, + "section": "Link reference definitions" + }, + { + "markdown": "[ΑΓΩ]: /φου\n\n[αγω]\n", + "html": "

αγω

\n", + "example": 206, + "start_line": 3352, + "end_line": 3358, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]: /url\n", + "html": "", + "example": 207, + "start_line": 3367, + "end_line": 3370, + "section": "Link reference definitions" + }, + { + "markdown": "[\nfoo\n]: /url\nbar\n", + "html": "

bar

\n", + "example": 208, + "start_line": 3375, + "end_line": 3382, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]: /url \"title\" ok\n", + "html": "

[foo]: /url "title" ok

\n", + "example": 209, + "start_line": 3388, + "end_line": 3392, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]: /url\n\"title\" ok\n", + "html": "

"title" ok

\n", + "example": 210, + "start_line": 3397, + "end_line": 3402, + "section": "Link reference definitions" + }, + { + "markdown": " [foo]: /url \"title\"\n\n[foo]\n", + "html": "
[foo]: /url "title"\n
\n

[foo]

\n", + "example": 211, + "start_line": 3408, + "end_line": 3416, + "section": "Link reference definitions" + }, + { + "markdown": "```\n[foo]: /url\n```\n\n[foo]\n", + "html": "
[foo]: /url\n
\n

[foo]

\n", + "example": 212, + "start_line": 3422, + "end_line": 3432, + "section": "Link reference definitions" + }, + { + "markdown": "Foo\n[bar]: /baz\n\n[bar]\n", + "html": "

Foo\n[bar]: /baz

\n

[bar]

\n", + "example": 213, + "start_line": 3437, + "end_line": 3446, + "section": "Link reference definitions" + }, + { + "markdown": "# [Foo]\n[foo]: /url\n> bar\n", + "html": "

Foo

\n
\n

bar

\n
\n", + "example": 214, + "start_line": 3452, + "end_line": 3461, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]: /url\nbar\n===\n[foo]\n", + "html": "

bar

\n

foo

\n", + "example": 215, + "start_line": 3463, + "end_line": 3471, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]: /url\n===\n[foo]\n", + "html": "

===\nfoo

\n", + "example": 216, + "start_line": 3473, + "end_line": 3480, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]: /foo-url \"foo\"\n[bar]: /bar-url\n \"bar\"\n[baz]: /baz-url\n\n[foo],\n[bar],\n[baz]\n", + "html": "

foo,\nbar,\nbaz

\n", + "example": 217, + "start_line": 3486, + "end_line": 3499, + "section": "Link reference definitions" + }, + { + "markdown": "[foo]\n\n> [foo]: /url\n", + "html": "

foo

\n
\n
\n", + "example": 218, + "start_line": 3507, + "end_line": 3515, + "section": "Link reference definitions" + }, + { + "markdown": "aaa\n\nbbb\n", + "html": "

aaa

\n

bbb

\n", + "example": 219, + "start_line": 3529, + "end_line": 3536, + "section": "Paragraphs" + }, + { + "markdown": "aaa\nbbb\n\nccc\nddd\n", + "html": "

aaa\nbbb

\n

ccc\nddd

\n", + "example": 220, + "start_line": 3541, + "end_line": 3552, + "section": "Paragraphs" + }, + { + "markdown": "aaa\n\n\nbbb\n", + "html": "

aaa

\n

bbb

\n", + "example": 221, + "start_line": 3557, + "end_line": 3565, + "section": "Paragraphs" + }, + { + "markdown": " aaa\n bbb\n", + "html": "

aaa\nbbb

\n", + "example": 222, + "start_line": 3570, + "end_line": 3576, + "section": "Paragraphs" + }, + { + "markdown": "aaa\n bbb\n ccc\n", + "html": "

aaa\nbbb\nccc

\n", + "example": 223, + "start_line": 3582, + "end_line": 3590, + "section": "Paragraphs" + }, + { + "markdown": " aaa\nbbb\n", + "html": "

aaa\nbbb

\n", + "example": 224, + "start_line": 3596, + "end_line": 3602, + "section": "Paragraphs" + }, + { + "markdown": " aaa\nbbb\n", + "html": "
aaa\n
\n

bbb

\n", + "example": 225, + "start_line": 3605, + "end_line": 3612, + "section": "Paragraphs" + }, + { + "markdown": "aaa \nbbb \n", + "html": "

aaa
\nbbb

\n", + "example": 226, + "start_line": 3619, + "end_line": 3625, + "section": "Paragraphs" + }, + { + "markdown": " \n\naaa\n \n\n# aaa\n\n \n", + "html": "

aaa

\n

aaa

\n", + "example": 227, + "start_line": 3636, + "end_line": 3648, + "section": "Blank lines" + }, + { + "markdown": "> # Foo\n> bar\n> baz\n", + "html": "
\n

Foo

\n

bar\nbaz

\n
\n", + "example": 228, + "start_line": 3704, + "end_line": 3714, + "section": "Block quotes" + }, + { + "markdown": "># Foo\n>bar\n> baz\n", + "html": "
\n

Foo

\n

bar\nbaz

\n
\n", + "example": 229, + "start_line": 3719, + "end_line": 3729, + "section": "Block quotes" + }, + { + "markdown": " > # Foo\n > bar\n > baz\n", + "html": "
\n

Foo

\n

bar\nbaz

\n
\n", + "example": 230, + "start_line": 3734, + "end_line": 3744, + "section": "Block quotes" + }, + { + "markdown": " > # Foo\n > bar\n > baz\n", + "html": "
> # Foo\n> bar\n> baz\n
\n", + "example": 231, + "start_line": 3749, + "end_line": 3758, + "section": "Block quotes" + }, + { + "markdown": "> # Foo\n> bar\nbaz\n", + "html": "
\n

Foo

\n

bar\nbaz

\n
\n", + "example": 232, + "start_line": 3764, + "end_line": 3774, + "section": "Block quotes" + }, + { + "markdown": "> bar\nbaz\n> foo\n", + "html": "
\n

bar\nbaz\nfoo

\n
\n", + "example": 233, + "start_line": 3780, + "end_line": 3790, + "section": "Block quotes" + }, + { + "markdown": "> foo\n---\n", + "html": "
\n

foo

\n
\n
\n", + "example": 234, + "start_line": 3804, + "end_line": 3812, + "section": "Block quotes" + }, + { + "markdown": "> - foo\n- bar\n", + "html": "
\n
    \n
  • foo
    • \n
\n
\n
    \n
  • bar
    • \n
\n", + "example": 235, + "start_line": 3824, + "end_line": 3836, + "section": "Block quotes" + }, + { + "markdown": "> foo\n bar\n", + "html": "
\n
foo\n
\n
\n
bar\n
\n", + "example": 236, + "start_line": 3842, + "end_line": 3852, + "section": "Block quotes" + }, + { + "markdown": "> ```\nfoo\n```\n", + "html": "
\n
\n
\n

foo

\n
\n", + "example": 237, + "start_line": 3855, + "end_line": 3865, + "section": "Block quotes" + }, + { + "markdown": "> foo\n - bar\n", + "html": "
\n

foo\n- bar

\n
\n", + "example": 238, + "start_line": 3871, + "end_line": 3879, + "section": "Block quotes" + }, + { + "markdown": ">\n", + "html": "
\n
\n", + "example": 239, + "start_line": 3895, + "end_line": 3900, + "section": "Block quotes" + }, + { + "markdown": ">\n> \n> \n", + "html": "
\n
\n", + "example": 240, + "start_line": 3903, + "end_line": 3910, + "section": "Block quotes" + }, + { + "markdown": ">\n> foo\n> \n", + "html": "
\n

foo

\n
\n", + "example": 241, + "start_line": 3915, + "end_line": 3923, + "section": "Block quotes" + }, + { + "markdown": "> foo\n\n> bar\n", + "html": "
\n

foo

\n
\n
\n

bar

\n
\n", + "example": 242, + "start_line": 3928, + "end_line": 3939, + "section": "Block quotes" + }, + { + "markdown": "> foo\n> bar\n", + "html": "
\n

foo\nbar

\n
\n", + "example": 243, + "start_line": 3950, + "end_line": 3958, + "section": "Block quotes" + }, + { + "markdown": "> foo\n>\n> bar\n", + "html": "
\n

foo

\n

bar

\n
\n", + "example": 244, + "start_line": 3963, + "end_line": 3972, + "section": "Block quotes" + }, + { + "markdown": "foo\n> bar\n", + "html": "

foo

\n
\n

bar

\n
\n", + "example": 245, + "start_line": 3977, + "end_line": 3985, + "section": "Block quotes" + }, + { + "markdown": "> aaa\n***\n> bbb\n", + "html": "
\n

aaa

\n
\n
\n
\n

bbb

\n
\n", + "example": 246, + "start_line": 3991, + "end_line": 4003, + "section": "Block quotes" + }, + { + "markdown": "> bar\nbaz\n", + "html": "
\n

bar\nbaz

\n
\n", + "example": 247, + "start_line": 4009, + "end_line": 4017, + "section": "Block quotes" + }, + { + "markdown": "> bar\n\nbaz\n", + "html": "
\n

bar

\n
\n

baz

\n", + "example": 248, + "start_line": 4020, + "end_line": 4029, + "section": "Block quotes" + }, + { + "markdown": "> bar\n>\nbaz\n", + "html": "
\n

bar

\n
\n

baz

\n", + "example": 249, + "start_line": 4032, + "end_line": 4041, + "section": "Block quotes" + }, + { + "markdown": "> > > foo\nbar\n", + "html": "
\n
\n
\n

foo\nbar

\n
\n
\n
\n", + "example": 250, + "start_line": 4048, + "end_line": 4060, + "section": "Block quotes" + }, + { + "markdown": ">>> foo\n> bar\n>>baz\n", + "html": "
\n
\n
\n

foo\nbar\nbaz

\n
\n
\n
\n", + "example": 251, + "start_line": 4063, + "end_line": 4077, + "section": "Block quotes" + }, + { + "markdown": "> code\n\n> not code\n", + "html": "
\n
code\n
\n
\n
\n

not code

\n
\n", + "example": 252, + "start_line": 4085, + "end_line": 4097, + "section": "Block quotes" + }, + { + "markdown": "A paragraph\nwith two lines.\n\n indented code\n\n> A block quote.\n", + "html": "

A paragraph\nwith two lines.

\n
indented code\n
\n
\n

A block quote.

\n
\n", + "example": 253, + "start_line": 4139, + "end_line": 4154, + "section": "List items" + }, + { + "markdown": "1. A paragraph\n with two lines.\n\n indented code\n\n > A block quote.\n", + "html": "
    \n
  1. \n

    A paragraph\nwith two lines.

    \n
    indented code\n
    \n
    \n

    A block quote.

    \n
    \n
    2. \n
\n", + "example": 254, + "start_line": 4161, + "end_line": 4180, + "section": "List items" + }, + { + "markdown": "- one\n\n two\n", + "html": "
    \n
  • one
    • \n
\n

two

\n", + "example": 255, + "start_line": 4194, + "end_line": 4203, + "section": "List items" + }, + { + "markdown": "- one\n\n two\n", + "html": "
    \n
  • \n

    one

    \n

    two

    \n
    • \n
\n", + "example": 256, + "start_line": 4206, + "end_line": 4217, + "section": "List items" + }, + { + "markdown": " - one\n\n two\n", + "html": "
    \n
  • one
    • \n
\n
 two\n
\n", + "example": 257, + "start_line": 4220, + "end_line": 4230, + "section": "List items" + }, + { + "markdown": " - one\n\n two\n", + "html": "
    \n
  • \n

    one

    \n

    two

    \n
    • \n
\n", + "example": 258, + "start_line": 4233, + "end_line": 4244, + "section": "List items" + }, + { + "markdown": " > > 1. one\n>>\n>> two\n", + "html": "
\n
\n
    \n
  1. \n

    one

    \n

    two

    \n
    2. \n
\n
\n
\n", + "example": 259, + "start_line": 4255, + "end_line": 4270, + "section": "List items" + }, + { + "markdown": ">>- one\n>>\n > > two\n", + "html": "
\n
\n
    \n
  • one
    • \n
\n

two

\n
\n
\n", + "example": 260, + "start_line": 4282, + "end_line": 4295, + "section": "List items" + }, + { + "markdown": "-one\n\n2.two\n", + "html": "

-one

\n

2.two

\n", + "example": 261, + "start_line": 4301, + "end_line": 4308, + "section": "List items" + }, + { + "markdown": "- foo\n\n\n bar\n", + "html": "
    \n
  • \n

    foo

    \n

    bar

    \n
    • \n
\n", + "example": 262, + "start_line": 4314, + "end_line": 4326, + "section": "List items" + }, + { + "markdown": "1. foo\n\n ```\n bar\n ```\n\n baz\n\n > bam\n", + "html": "
    \n
  1. \n

    foo

    \n
    bar\n
    \n

    baz

    \n
    \n

    bam

    \n
    \n
    2. \n
\n", + "example": 263, + "start_line": 4331, + "end_line": 4353, + "section": "List items" + }, + { + "markdown": "- Foo\n\n bar\n\n\n baz\n", + "html": "
    \n
  • \n

    Foo

    \n
    bar\n\n\nbaz\n
    \n
    • \n
\n", + "example": 264, + "start_line": 4359, + "end_line": 4377, + "section": "List items" + }, + { + "markdown": "123456789. ok\n", + "html": "
    \n
  1. ok
    2. \n
\n", + "example": 265, + "start_line": 4381, + "end_line": 4387, + "section": "List items" + }, + { + "markdown": "1234567890. not ok\n", + "html": "

1234567890. not ok

\n", + "example": 266, + "start_line": 4390, + "end_line": 4394, + "section": "List items" + }, + { + "markdown": "0. ok\n", + "html": "
    \n
  1. ok
    2. \n
\n", + "example": 267, + "start_line": 4399, + "end_line": 4405, + "section": "List items" + }, + { + "markdown": "003. ok\n", + "html": "
    \n
  1. ok
    2. \n
\n", + "example": 268, + "start_line": 4408, + "end_line": 4414, + "section": "List items" + }, + { + "markdown": "-1. not ok\n", + "html": "

-1. not ok

\n", + "example": 269, + "start_line": 4419, + "end_line": 4423, + "section": "List items" + }, + { + "markdown": "- foo\n\n bar\n", + "html": "
    \n
  • \n

    foo

    \n
    bar\n
    \n
    • \n
\n", + "example": 270, + "start_line": 4442, + "end_line": 4454, + "section": "List items" + }, + { + "markdown": " 10. foo\n\n bar\n", + "html": "
    \n
  1. \n

    foo

    \n
    bar\n
    \n
    2. \n
\n", + "example": 271, + "start_line": 4459, + "end_line": 4471, + "section": "List items" + }, + { + "markdown": " indented code\n\nparagraph\n\n more code\n", + "html": "
indented code\n
\n

paragraph

\n
more code\n
\n", + "example": 272, + "start_line": 4478, + "end_line": 4490, + "section": "List items" + }, + { + "markdown": "1. indented code\n\n paragraph\n\n more code\n", + "html": "
    \n
  1. \n
    indented code\n
    \n

    paragraph

    \n
    more code\n
    \n
    2. \n
\n", + "example": 273, + "start_line": 4493, + "end_line": 4509, + "section": "List items" + }, + { + "markdown": "1. indented code\n\n paragraph\n\n more code\n", + "html": "
    \n
  1. \n
     indented code\n
    \n

    paragraph

    \n
    more code\n
    \n
    2. \n
\n", + "example": 274, + "start_line": 4515, + "end_line": 4531, + "section": "List items" + }, + { + "markdown": " foo\n\nbar\n", + "html": "

foo

\n

bar

\n", + "example": 275, + "start_line": 4542, + "end_line": 4549, + "section": "List items" + }, + { + "markdown": "- foo\n\n bar\n", + "html": "
    \n
  • foo
    • \n
\n

bar

\n", + "example": 276, + "start_line": 4552, + "end_line": 4561, + "section": "List items" + }, + { + "markdown": "- foo\n\n bar\n", + "html": "
    \n
  • \n

    foo

    \n

    bar

    \n
    • \n
\n", + "example": 277, + "start_line": 4569, + "end_line": 4580, + "section": "List items" + }, + { + "markdown": "-\n foo\n-\n ```\n bar\n ```\n-\n baz\n", + "html": "
    \n
  • foo
    • \n
  • \n
    bar\n
    \n
    • \n
  • \n
    baz\n
    \n
    • \n
\n", + "example": 278, + "start_line": 4596, + "end_line": 4617, + "section": "List items" + }, + { + "markdown": "- \n foo\n", + "html": "
    \n
  • foo
    • \n
\n", + "example": 279, + "start_line": 4622, + "end_line": 4629, + "section": "List items" + }, + { + "markdown": "-\n\n foo\n", + "html": "
    \n
  • \n
\n

foo

\n", + "example": 280, + "start_line": 4636, + "end_line": 4645, + "section": "List items" + }, + { + "markdown": "- foo\n-\n- bar\n", + "html": "
    \n
  • foo
    • \n
  • \n
  • bar
    • \n
\n", + "example": 281, + "start_line": 4650, + "end_line": 4660, + "section": "List items" + }, + { + "markdown": "- foo\n- \n- bar\n", + "html": "
    \n
  • foo
    • \n
  • \n
  • bar
    • \n
\n", + "example": 282, + "start_line": 4665, + "end_line": 4675, + "section": "List items" + }, + { + "markdown": "1. foo\n2.\n3. bar\n", + "html": "
    \n
  1. foo
    2. \n
  2. \n
  3. bar
    4. \n
\n", + "example": 283, + "start_line": 4680, + "end_line": 4690, + "section": "List items" + }, + { + "markdown": "*\n", + "html": "
    \n
  • \n
\n", + "example": 284, + "start_line": 4695, + "end_line": 4701, + "section": "List items" + }, + { + "markdown": "foo\n*\n\nfoo\n1.\n", + "html": "

foo\n*

\n

foo\n1.

\n", + "example": 285, + "start_line": 4705, + "end_line": 4716, + "section": "List items" + }, + { + "markdown": " 1. A paragraph\n with two lines.\n\n indented code\n\n > A block quote.\n", + "html": "
    \n
  1. \n

    A paragraph\nwith two lines.

    \n
    indented code\n
    \n
    \n

    A block quote.

    \n
    \n
    2. \n
\n", + "example": 286, + "start_line": 4727, + "end_line": 4746, + "section": "List items" + }, + { + "markdown": " 1. A paragraph\n with two lines.\n\n indented code\n\n > A block quote.\n", + "html": "
    \n
  1. \n

    A paragraph\nwith two lines.

    \n
    indented code\n
    \n
    \n

    A block quote.

    \n
    \n
    2. \n
\n", + "example": 287, + "start_line": 4751, + "end_line": 4770, + "section": "List items" + }, + { + "markdown": " 1. A paragraph\n with two lines.\n\n indented code\n\n > A block quote.\n", + "html": "
    \n
  1. \n

    A paragraph\nwith two lines.

    \n
    indented code\n
    \n
    \n

    A block quote.

    \n
    \n
    2. \n
\n", + "example": 288, + "start_line": 4775, + "end_line": 4794, + "section": "List items" + }, + { + "markdown": " 1. A paragraph\n with two lines.\n\n indented code\n\n > A block quote.\n", + "html": "
1.  A paragraph\n    with two lines.\n\n        indented code\n\n    > A block quote.\n
\n", + "example": 289, + "start_line": 4799, + "end_line": 4814, + "section": "List items" + }, + { + "markdown": " 1. A paragraph\nwith two lines.\n\n indented code\n\n > A block quote.\n", + "html": "
    \n
  1. \n

    A paragraph\nwith two lines.

    \n
    indented code\n
    \n
    \n

    A block quote.

    \n
    \n
    2. \n
\n", + "example": 290, + "start_line": 4829, + "end_line": 4848, + "section": "List items" + }, + { + "markdown": " 1. A paragraph\n with two lines.\n", + "html": "
    \n
  1. A paragraph\nwith two lines.
    2. \n
\n", + "example": 291, + "start_line": 4853, + "end_line": 4861, + "section": "List items" + }, + { + "markdown": "> 1. > Blockquote\ncontinued here.\n", + "html": "
\n
    \n
  1. \n
    \n

    Blockquote\ncontinued here.

    \n
    \n
    2. \n
\n
\n", + "example": 292, + "start_line": 4866, + "end_line": 4880, + "section": "List items" + }, + { + "markdown": "> 1. > Blockquote\n> continued here.\n", + "html": "
\n
    \n
  1. \n
    \n

    Blockquote\ncontinued here.

    \n
    \n
    2. \n
\n
\n", + "example": 293, + "start_line": 4883, + "end_line": 4897, + "section": "List items" + }, + { + "markdown": "- foo\n - bar\n - baz\n - boo\n", + "html": "
    \n
  • foo\n
      \n
    • bar\n
        \n
      • baz\n
          \n
        • boo
          • \n
        \n
        • \n
      \n
      • \n
    \n
    • \n
\n", + "example": 294, + "start_line": 4911, + "end_line": 4932, + "section": "List items" + }, + { + "markdown": "- foo\n - bar\n - baz\n - boo\n", + "html": "
    \n
  • foo
    • \n
  • bar
    • \n
  • baz
    • \n
  • boo
    • \n
\n", + "example": 295, + "start_line": 4937, + "end_line": 4949, + "section": "List items" + }, + { + "markdown": "10) foo\n - bar\n", + "html": "
    \n
  1. foo\n
      \n
    • bar
      • \n
    \n
    2. \n
\n", + "example": 296, + "start_line": 4954, + "end_line": 4965, + "section": "List items" + }, + { + "markdown": "10) foo\n - bar\n", + "html": "
    \n
  1. foo
    2. \n
\n
    \n
  • bar
    • \n
\n", + "example": 297, + "start_line": 4970, + "end_line": 4980, + "section": "List items" + }, + { + "markdown": "- - foo\n", + "html": "
    \n
  • \n
      \n
    • foo
      • \n
    \n
    • \n
\n", + "example": 298, + "start_line": 4985, + "end_line": 4995, + "section": "List items" + }, + { + "markdown": "1. - 2. foo\n", + "html": "
    \n
  1. \n
      \n
    • \n
        \n
      1. foo
        2. \n
      \n
      • \n
    \n
    2. \n
\n", + "example": 299, + "start_line": 4998, + "end_line": 5012, + "section": "List items" + }, + { + "markdown": "- # Foo\n- Bar\n ---\n baz\n", + "html": "
    \n
  • \n

    Foo

    \n
    • \n
  • \n

    Bar

    \nbaz
    • \n
\n", + "example": 300, + "start_line": 5017, + "end_line": 5031, + "section": "List items" + }, + { + "markdown": "- foo\n- bar\n+ baz\n", + "html": "
    \n
  • foo
    • \n
  • bar
    • \n
\n
    \n
  • baz
    • \n
\n", + "example": 301, + "start_line": 5253, + "end_line": 5265, + "section": "Lists" + }, + { + "markdown": "1. foo\n2. bar\n3) baz\n", + "html": "
    \n
  1. foo
    2. \n
  2. bar
    3. \n
\n
    \n
  1. baz
    2. \n
\n", + "example": 302, + "start_line": 5268, + "end_line": 5280, + "section": "Lists" + }, + { + "markdown": "Foo\n- bar\n- baz\n", + "html": "

Foo

\n
    \n
  • bar
    • \n
  • baz
    • \n
\n", + "example": 303, + "start_line": 5287, + "end_line": 5297, + "section": "Lists" + }, + { + "markdown": "The number of windows in my house is\n14. The number of doors is 6.\n", + "html": "

The number of windows in my house is\n14. The number of doors is 6.

\n", + "example": 304, + "start_line": 5364, + "end_line": 5370, + "section": "Lists" + }, + { + "markdown": "The number of windows in my house is\n1. The number of doors is 6.\n", + "html": "

The number of windows in my house is

\n
    \n
  1. The number of doors is 6.
    2. \n
\n", + "example": 305, + "start_line": 5374, + "end_line": 5382, + "section": "Lists" + }, + { + "markdown": "- foo\n\n- bar\n\n\n- baz\n", + "html": "
    \n
  • \n

    foo

    \n
    • \n
  • \n

    bar

    \n
    • \n
  • \n

    baz

    \n
    • \n
\n", + "example": 306, + "start_line": 5388, + "end_line": 5407, + "section": "Lists" + }, + { + "markdown": "- foo\n - bar\n - baz\n\n\n bim\n", + "html": "
    \n
  • foo\n
      \n
    • bar\n
        \n
      • \n

        baz

        \n

        bim

        \n
        • \n
      \n
      • \n
    \n
    • \n
\n", + "example": 307, + "start_line": 5409, + "end_line": 5431, + "section": "Lists" + }, + { + "markdown": "- foo\n- bar\n\n\n\n- baz\n- bim\n", + "html": "
    \n
  • foo
    • \n
  • bar
    • \n
\n\n
    \n
  • baz
    • \n
  • bim
    • \n
\n", + "example": 308, + "start_line": 5439, + "end_line": 5457, + "section": "Lists" + }, + { + "markdown": "- foo\n\n notcode\n\n- foo\n\n\n\n code\n", + "html": "
    \n
  • \n

    foo

    \n

    notcode

    \n
    • \n
  • \n

    foo

    \n
    • \n
\n\n
code\n
\n", + "example": 309, + "start_line": 5460, + "end_line": 5483, + "section": "Lists" + }, + { + "markdown": "- a\n - b\n - c\n - d\n - e\n - f\n- g\n", + "html": "
    \n
  • a
    • \n
  • b
    • \n
  • c
    • \n
  • d
    • \n
  • e
    • \n
  • f
    • \n
  • g
    • \n
\n", + "example": 310, + "start_line": 5491, + "end_line": 5509, + "section": "Lists" + }, + { + "markdown": "1. a\n\n 2. b\n\n 3. c\n", + "html": "
    \n
  1. \n

    a

    \n
    2. \n
  2. \n

    b

    \n
    3. \n
  3. \n

    c

    \n
    4. \n
\n", + "example": 311, + "start_line": 5512, + "end_line": 5530, + "section": "Lists" + }, + { + "markdown": "- a\n - b\n - c\n - d\n - e\n", + "html": "
    \n
  • a
    • \n
  • b
    • \n
  • c
    • \n
  • d\n- e
    • \n
\n", + "example": 312, + "start_line": 5536, + "end_line": 5550, + "section": "Lists" + }, + { + "markdown": "1. a\n\n 2. b\n\n 3. c\n", + "html": "
    \n
  1. \n

    a

    \n
    2. \n
  2. \n

    b

    \n
    3. \n
\n
3. c\n
\n", + "example": 313, + "start_line": 5556, + "end_line": 5573, + "section": "Lists" + }, + { + "markdown": "- a\n- b\n\n- c\n", + "html": "
    \n
  • \n

    a

    \n
    • \n
  • \n

    b

    \n
    • \n
  • \n

    c

    \n
    • \n
\n", + "example": 314, + "start_line": 5579, + "end_line": 5596, + "section": "Lists" + }, + { + "markdown": "* a\n*\n\n* c\n", + "html": "
    \n
  • \n

    a

    \n
    • \n
  • \n
  • \n

    c

    \n
    • \n
\n", + "example": 315, + "start_line": 5601, + "end_line": 5616, + "section": "Lists" + }, + { + "markdown": "- a\n- b\n\n c\n- d\n", + "html": "
    \n
  • \n

    a

    \n
    • \n
  • \n

    b

    \n

    c

    \n
    • \n
  • \n

    d

    \n
    • \n
\n", + "example": 316, + "start_line": 5623, + "end_line": 5642, + "section": "Lists" + }, + { + "markdown": "- a\n- b\n\n [ref]: /url\n- d\n", + "html": "
    \n
  • \n

    a

    \n
    • \n
  • \n

    b

    \n
    • \n
  • \n

    d

    \n
    • \n
\n", + "example": 317, + "start_line": 5645, + "end_line": 5663, + "section": "Lists" + }, + { + "markdown": "- a\n- ```\n b\n\n\n ```\n- c\n", + "html": "
    \n
  • a
    • \n
  • \n
    b\n\n\n
    \n
    • \n
  • c
    • \n
\n", + "example": 318, + "start_line": 5668, + "end_line": 5687, + "section": "Lists" + }, + { + "markdown": "- a\n - b\n\n c\n- d\n", + "html": "
    \n
  • a\n
      \n
    • \n

      b

      \n

      c

      \n
      • \n
    \n
    • \n
  • d
    • \n
\n", + "example": 319, + "start_line": 5694, + "end_line": 5712, + "section": "Lists" + }, + { + "markdown": "* a\n > b\n >\n* c\n", + "html": "
    \n
  • a\n
    \n

    b

    \n
    \n
    • \n
  • c
    • \n
\n", + "example": 320, + "start_line": 5718, + "end_line": 5732, + "section": "Lists" + }, + { + "markdown": "- a\n > b\n ```\n c\n ```\n- d\n", + "html": "
    \n
  • a\n
    \n

    b

    \n
    \n
    c\n
    \n
    • \n
  • d
    • \n
\n", + "example": 321, + "start_line": 5738, + "end_line": 5756, + "section": "Lists" + }, + { + "markdown": "- a\n", + "html": "
    \n
  • a
    • \n
\n", + "example": 322, + "start_line": 5761, + "end_line": 5767, + "section": "Lists" + }, + { + "markdown": "- a\n - b\n", + "html": "
    \n
  • a\n
      \n
    • b
      • \n
    \n
    • \n
\n", + "example": 323, + "start_line": 5770, + "end_line": 5781, + "section": "Lists" + }, + { + "markdown": "1. ```\n foo\n ```\n\n bar\n", + "html": "
    \n
  1. \n
    foo\n
    \n

    bar

    \n
    2. \n
\n", + "example": 324, + "start_line": 5787, + "end_line": 5801, + "section": "Lists" + }, + { + "markdown": "* foo\n * bar\n\n baz\n", + "html": "
    \n
  • \n

    foo

    \n
      \n
    • bar
      • \n
    \n

    baz

    \n
    • \n
\n", + "example": 325, + "start_line": 5806, + "end_line": 5821, + "section": "Lists" + }, + { + "markdown": "- a\n - b\n - c\n\n- d\n - e\n - f\n", + "html": "
    \n
  • \n

    a

    \n
      \n
    • b
      • \n
    • c
      • \n
    \n
    • \n
  • \n

    d

    \n
      \n
    • e
      • \n
    • f
      • \n
    \n
    • \n
\n", + "example": 326, + "start_line": 5824, + "end_line": 5849, + "section": "Lists" + }, + { + "markdown": "`hi`lo`\n", + "html": "

hilo`

\n", + "example": 327, + "start_line": 5858, + "end_line": 5862, + "section": "Inlines" + }, + { + "markdown": "`foo`\n", + "html": "

foo

\n", + "example": 328, + "start_line": 5890, + "end_line": 5894, + "section": "Code spans" + }, + { + "markdown": "`` foo ` bar ``\n", + "html": "

foo ` bar

\n", + "example": 329, + "start_line": 5901, + "end_line": 5905, + "section": "Code spans" + }, + { + "markdown": "` `` `\n", + "html": "

``

\n", + "example": 330, + "start_line": 5911, + "end_line": 5915, + "section": "Code spans" + }, + { + "markdown": "` `` `\n", + "html": "

``

\n", + "example": 331, + "start_line": 5919, + "end_line": 5923, + "section": "Code spans" + }, + { + "markdown": "` a`\n", + "html": "

a

\n", + "example": 332, + "start_line": 5928, + "end_line": 5932, + "section": "Code spans" + }, + { + "markdown": "` b `\n", + "html": "

 b 

\n", + "example": 333, + "start_line": 5937, + "end_line": 5941, + "section": "Code spans" + }, + { + "markdown": "` `\n` `\n", + "html": "

 \n

\n", + "example": 334, + "start_line": 5945, + "end_line": 5951, + "section": "Code spans" + }, + { + "markdown": "``\nfoo\nbar \nbaz\n``\n", + "html": "

foo bar baz

\n", + "example": 335, + "start_line": 5956, + "end_line": 5964, + "section": "Code spans" + }, + { + "markdown": "``\nfoo \n``\n", + "html": "

foo

\n", + "example": 336, + "start_line": 5966, + "end_line": 5972, + "section": "Code spans" + }, + { + "markdown": "`foo bar \nbaz`\n", + "html": "

foo bar baz

\n", + "example": 337, + "start_line": 5977, + "end_line": 5982, + "section": "Code spans" + }, + { + "markdown": "`foo\\`bar`\n", + "html": "

foo\\bar`

\n", + "example": 338, + "start_line": 5994, + "end_line": 5998, + "section": "Code spans" + }, + { + "markdown": "``foo`bar``\n", + "html": "

foo`bar

\n", + "example": 339, + "start_line": 6005, + "end_line": 6009, + "section": "Code spans" + }, + { + "markdown": "` foo `` bar `\n", + "html": "

foo `` bar

\n", + "example": 340, + "start_line": 6011, + "end_line": 6015, + "section": "Code spans" + }, + { + "markdown": "*foo`*`\n", + "html": "

*foo*

\n", + "example": 341, + "start_line": 6023, + "end_line": 6027, + "section": "Code spans" + }, + { + "markdown": "[not a `link](/foo`)\n", + "html": "

[not a link](/foo)

\n", + "example": 342, + "start_line": 6032, + "end_line": 6036, + "section": "Code spans" + }, + { + "markdown": "``\n", + "html": "

<a href="">`

\n", + "example": 343, + "start_line": 6042, + "end_line": 6046, + "section": "Code spans" + }, + { + "markdown": "`\n", + "html": "

`

\n", + "example": 344, + "start_line": 6051, + "end_line": 6055, + "section": "Code spans" + }, + { + "markdown": "``\n", + "html": "

<http://foo.bar.baz>`

\n", + "example": 345, + "start_line": 6060, + "end_line": 6064, + "section": "Code spans" + }, + { + "markdown": "`\n", + "html": "

http://foo.bar.`baz`

\n", + "example": 346, + "start_line": 6069, + "end_line": 6073, + "section": "Code spans" + }, + { + "markdown": "```foo``\n", + "html": "

```foo``

\n", + "example": 347, + "start_line": 6079, + "end_line": 6083, + "section": "Code spans" + }, + { + "markdown": "`foo\n", + "html": "

`foo

\n", + "example": 348, + "start_line": 6086, + "end_line": 6090, + "section": "Code spans" + }, + { + "markdown": "`foo``bar``\n", + "html": "

`foobar

\n", + "example": 349, + "start_line": 6095, + "end_line": 6099, + "section": "Code spans" + }, + { + "markdown": "*foo bar*\n", + "html": "

foo bar

\n", + "example": 350, + "start_line": 6312, + "end_line": 6316, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "a * foo bar*\n", + "html": "

a * foo bar*

\n", + "example": 351, + "start_line": 6322, + "end_line": 6326, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "a*\"foo\"*\n", + "html": "

a*"foo"*

\n", + "example": 352, + "start_line": 6333, + "end_line": 6337, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "* a *\n", + "html": "

* a *

\n", + "example": 353, + "start_line": 6342, + "end_line": 6346, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo*bar*\n", + "html": "

foobar

\n", + "example": 354, + "start_line": 6351, + "end_line": 6355, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "5*6*78\n", + "html": "

5678

\n", + "example": 355, + "start_line": 6358, + "end_line": 6362, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_foo bar_\n", + "html": "

foo bar

\n", + "example": 356, + "start_line": 6367, + "end_line": 6371, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_ foo bar_\n", + "html": "

_ foo bar_

\n", + "example": 357, + "start_line": 6377, + "end_line": 6381, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "a_\"foo\"_\n", + "html": "

a_"foo"_

\n", + "example": 358, + "start_line": 6387, + "end_line": 6391, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo_bar_\n", + "html": "

foo_bar_

\n", + "example": 359, + "start_line": 6396, + "end_line": 6400, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "5_6_78\n", + "html": "

5_6_78

\n", + "example": 360, + "start_line": 6403, + "end_line": 6407, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "пристаням_стремятся_\n", + "html": "

пристаням_стремятся_

\n", + "example": 361, + "start_line": 6410, + "end_line": 6414, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "aa_\"bb\"_cc\n", + "html": "

aa_"bb"_cc

\n", + "example": 362, + "start_line": 6420, + "end_line": 6424, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo-_(bar)_\n", + "html": "

foo-(bar)

\n", + "example": 363, + "start_line": 6431, + "end_line": 6435, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_foo*\n", + "html": "

_foo*

\n", + "example": 364, + "start_line": 6443, + "end_line": 6447, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo bar *\n", + "html": "

*foo bar *

\n", + "example": 365, + "start_line": 6453, + "end_line": 6457, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo bar\n*\n", + "html": "

*foo bar\n*

\n", + "example": 366, + "start_line": 6462, + "end_line": 6468, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*(*foo)\n", + "html": "

*(*foo)

\n", + "example": 367, + "start_line": 6475, + "end_line": 6479, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*(*foo*)*\n", + "html": "

(foo)

\n", + "example": 368, + "start_line": 6485, + "end_line": 6489, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo*bar\n", + "html": "

foobar

\n", + "example": 369, + "start_line": 6494, + "end_line": 6498, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_foo bar _\n", + "html": "

_foo bar _

\n", + "example": 370, + "start_line": 6507, + "end_line": 6511, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_(_foo)\n", + "html": "

_(_foo)

\n", + "example": 371, + "start_line": 6517, + "end_line": 6521, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_(_foo_)_\n", + "html": "

(foo)

\n", + "example": 372, + "start_line": 6526, + "end_line": 6530, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_foo_bar\n", + "html": "

_foo_bar

\n", + "example": 373, + "start_line": 6535, + "end_line": 6539, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_пристаням_стремятся\n", + "html": "

_пристаням_стремятся

\n", + "example": 374, + "start_line": 6542, + "end_line": 6546, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_foo_bar_baz_\n", + "html": "

foo_bar_baz

\n", + "example": 375, + "start_line": 6549, + "end_line": 6553, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_(bar)_.\n", + "html": "

(bar).

\n", + "example": 376, + "start_line": 6560, + "end_line": 6564, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo bar**\n", + "html": "

foo bar

\n", + "example": 377, + "start_line": 6569, + "end_line": 6573, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "** foo bar**\n", + "html": "

** foo bar**

\n", + "example": 378, + "start_line": 6579, + "end_line": 6583, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "a**\"foo\"**\n", + "html": "

a**"foo"**

\n", + "example": 379, + "start_line": 6590, + "end_line": 6594, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo**bar**\n", + "html": "

foobar

\n", + "example": 380, + "start_line": 6599, + "end_line": 6603, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__foo bar__\n", + "html": "

foo bar

\n", + "example": 381, + "start_line": 6608, + "end_line": 6612, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__ foo bar__\n", + "html": "

__ foo bar__

\n", + "example": 382, + "start_line": 6618, + "end_line": 6622, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__\nfoo bar__\n", + "html": "

__\nfoo bar__

\n", + "example": 383, + "start_line": 6626, + "end_line": 6632, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "a__\"foo\"__\n", + "html": "

a__"foo"__

\n", + "example": 384, + "start_line": 6638, + "end_line": 6642, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo__bar__\n", + "html": "

foo__bar__

\n", + "example": 385, + "start_line": 6647, + "end_line": 6651, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "5__6__78\n", + "html": "

5__6__78

\n", + "example": 386, + "start_line": 6654, + "end_line": 6658, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "пристаням__стремятся__\n", + "html": "

пристаням__стремятся__

\n", + "example": 387, + "start_line": 6661, + "end_line": 6665, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__foo, __bar__, baz__\n", + "html": "

foo, bar, baz

\n", + "example": 388, + "start_line": 6668, + "end_line": 6672, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo-__(bar)__\n", + "html": "

foo-(bar)

\n", + "example": 389, + "start_line": 6679, + "end_line": 6683, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo bar **\n", + "html": "

**foo bar **

\n", + "example": 390, + "start_line": 6692, + "end_line": 6696, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**(**foo)\n", + "html": "

**(**foo)

\n", + "example": 391, + "start_line": 6705, + "end_line": 6709, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*(**foo**)*\n", + "html": "

(foo)

\n", + "example": 392, + "start_line": 6715, + "end_line": 6719, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**Gomphocarpus (*Gomphocarpus physocarpus*, syn.\n*Asclepias physocarpa*)**\n", + "html": "

Gomphocarpus (Gomphocarpus physocarpus, syn.\nAsclepias physocarpa)

\n", + "example": 393, + "start_line": 6722, + "end_line": 6728, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo \"*bar*\" foo**\n", + "html": "

foo "bar" foo

\n", + "example": 394, + "start_line": 6731, + "end_line": 6735, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo**bar\n", + "html": "

foobar

\n", + "example": 395, + "start_line": 6740, + "end_line": 6744, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__foo bar __\n", + "html": "

__foo bar __

\n", + "example": 396, + "start_line": 6752, + "end_line": 6756, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__(__foo)\n", + "html": "

__(__foo)

\n", + "example": 397, + "start_line": 6762, + "end_line": 6766, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_(__foo__)_\n", + "html": "

(foo)

\n", + "example": 398, + "start_line": 6772, + "end_line": 6776, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__foo__bar\n", + "html": "

__foo__bar

\n", + "example": 399, + "start_line": 6781, + "end_line": 6785, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__пристаням__стремятся\n", + "html": "

__пристаням__стремятся

\n", + "example": 400, + "start_line": 6788, + "end_line": 6792, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__foo__bar__baz__\n", + "html": "

foo__bar__baz

\n", + "example": 401, + "start_line": 6795, + "end_line": 6799, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__(bar)__.\n", + "html": "

(bar).

\n", + "example": 402, + "start_line": 6806, + "end_line": 6810, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo [bar](/url)*\n", + "html": "

foo bar

\n", + "example": 403, + "start_line": 6818, + "end_line": 6822, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo\nbar*\n", + "html": "

foo\nbar

\n", + "example": 404, + "start_line": 6825, + "end_line": 6831, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_foo __bar__ baz_\n", + "html": "

foo bar baz

\n", + "example": 405, + "start_line": 6837, + "end_line": 6841, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_foo _bar_ baz_\n", + "html": "

foo bar baz

\n", + "example": 406, + "start_line": 6844, + "end_line": 6848, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__foo_ bar_\n", + "html": "

foo bar

\n", + "example": 407, + "start_line": 6851, + "end_line": 6855, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo *bar**\n", + "html": "

foo bar

\n", + "example": 408, + "start_line": 6858, + "end_line": 6862, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo **bar** baz*\n", + "html": "

foo bar baz

\n", + "example": 409, + "start_line": 6865, + "end_line": 6869, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo**bar**baz*\n", + "html": "

foobarbaz

\n", + "example": 410, + "start_line": 6871, + "end_line": 6875, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo**bar*\n", + "html": "

foo**bar

\n", + "example": 411, + "start_line": 6895, + "end_line": 6899, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "***foo** bar*\n", + "html": "

foo bar

\n", + "example": 412, + "start_line": 6908, + "end_line": 6912, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo **bar***\n", + "html": "

foo bar

\n", + "example": 413, + "start_line": 6915, + "end_line": 6919, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo**bar***\n", + "html": "

foobar

\n", + "example": 414, + "start_line": 6922, + "end_line": 6926, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo***bar***baz\n", + "html": "

foobarbaz

\n", + "example": 415, + "start_line": 6933, + "end_line": 6937, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo******bar*********baz\n", + "html": "

foobar***baz

\n", + "example": 416, + "start_line": 6939, + "end_line": 6943, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo **bar *baz* bim** bop*\n", + "html": "

foo bar baz bim bop

\n", + "example": 417, + "start_line": 6948, + "end_line": 6952, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo [*bar*](/url)*\n", + "html": "

foo bar

\n", + "example": 418, + "start_line": 6955, + "end_line": 6959, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "** is not an empty emphasis\n", + "html": "

** is not an empty emphasis

\n", + "example": 419, + "start_line": 6964, + "end_line": 6968, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**** is not an empty strong emphasis\n", + "html": "

**** is not an empty strong emphasis

\n", + "example": 420, + "start_line": 6971, + "end_line": 6975, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo [bar](/url)**\n", + "html": "

foo bar

\n", + "example": 421, + "start_line": 6984, + "end_line": 6988, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo\nbar**\n", + "html": "

foo\nbar

\n", + "example": 422, + "start_line": 6991, + "end_line": 6997, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__foo _bar_ baz__\n", + "html": "

foo bar baz

\n", + "example": 423, + "start_line": 7003, + "end_line": 7007, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__foo __bar__ baz__\n", + "html": "

foo bar baz

\n", + "example": 424, + "start_line": 7010, + "end_line": 7014, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "____foo__ bar__\n", + "html": "

foo bar

\n", + "example": 425, + "start_line": 7017, + "end_line": 7021, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo **bar****\n", + "html": "

foo bar

\n", + "example": 426, + "start_line": 7024, + "end_line": 7028, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo *bar* baz**\n", + "html": "

foo bar baz

\n", + "example": 427, + "start_line": 7031, + "end_line": 7035, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo*bar*baz**\n", + "html": "

foobarbaz

\n", + "example": 428, + "start_line": 7038, + "end_line": 7042, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "***foo* bar**\n", + "html": "

foo bar

\n", + "example": 429, + "start_line": 7045, + "end_line": 7049, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo *bar***\n", + "html": "

foo bar

\n", + "example": 430, + "start_line": 7052, + "end_line": 7056, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo *bar **baz**\nbim* bop**\n", + "html": "

foo bar baz\nbim bop

\n", + "example": 431, + "start_line": 7061, + "end_line": 7067, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo [*bar*](/url)**\n", + "html": "

foo bar

\n", + "example": 432, + "start_line": 7070, + "end_line": 7074, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__ is not an empty emphasis\n", + "html": "

__ is not an empty emphasis

\n", + "example": 433, + "start_line": 7079, + "end_line": 7083, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "____ is not an empty strong emphasis\n", + "html": "

____ is not an empty strong emphasis

\n", + "example": 434, + "start_line": 7086, + "end_line": 7090, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo ***\n", + "html": "

foo ***

\n", + "example": 435, + "start_line": 7096, + "end_line": 7100, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo *\\**\n", + "html": "

foo *

\n", + "example": 436, + "start_line": 7103, + "end_line": 7107, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo *_*\n", + "html": "

foo _

\n", + "example": 437, + "start_line": 7110, + "end_line": 7114, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo *****\n", + "html": "

foo *****

\n", + "example": 438, + "start_line": 7117, + "end_line": 7121, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo **\\***\n", + "html": "

foo *

\n", + "example": 439, + "start_line": 7124, + "end_line": 7128, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo **_**\n", + "html": "

foo _

\n", + "example": 440, + "start_line": 7131, + "end_line": 7135, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo*\n", + "html": "

*foo

\n", + "example": 441, + "start_line": 7142, + "end_line": 7146, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo**\n", + "html": "

foo*

\n", + "example": 442, + "start_line": 7149, + "end_line": 7153, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "***foo**\n", + "html": "

*foo

\n", + "example": 443, + "start_line": 7156, + "end_line": 7160, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "****foo*\n", + "html": "

***foo

\n", + "example": 444, + "start_line": 7163, + "end_line": 7167, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo***\n", + "html": "

foo*

\n", + "example": 445, + "start_line": 7170, + "end_line": 7174, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo****\n", + "html": "

foo***

\n", + "example": 446, + "start_line": 7177, + "end_line": 7181, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo ___\n", + "html": "

foo ___

\n", + "example": 447, + "start_line": 7187, + "end_line": 7191, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo _\\__\n", + "html": "

foo _

\n", + "example": 448, + "start_line": 7194, + "end_line": 7198, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo _*_\n", + "html": "

foo *

\n", + "example": 449, + "start_line": 7201, + "end_line": 7205, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo _____\n", + "html": "

foo _____

\n", + "example": 450, + "start_line": 7208, + "end_line": 7212, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo __\\___\n", + "html": "

foo _

\n", + "example": 451, + "start_line": 7215, + "end_line": 7219, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "foo __*__\n", + "html": "

foo *

\n", + "example": 452, + "start_line": 7222, + "end_line": 7226, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__foo_\n", + "html": "

_foo

\n", + "example": 453, + "start_line": 7229, + "end_line": 7233, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_foo__\n", + "html": "

foo_

\n", + "example": 454, + "start_line": 7240, + "end_line": 7244, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "___foo__\n", + "html": "

_foo

\n", + "example": 455, + "start_line": 7247, + "end_line": 7251, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "____foo_\n", + "html": "

___foo

\n", + "example": 456, + "start_line": 7254, + "end_line": 7258, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__foo___\n", + "html": "

foo_

\n", + "example": 457, + "start_line": 7261, + "end_line": 7265, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_foo____\n", + "html": "

foo___

\n", + "example": 458, + "start_line": 7268, + "end_line": 7272, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo**\n", + "html": "

foo

\n", + "example": 459, + "start_line": 7278, + "end_line": 7282, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*_foo_*\n", + "html": "

foo

\n", + "example": 460, + "start_line": 7285, + "end_line": 7289, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__foo__\n", + "html": "

foo

\n", + "example": 461, + "start_line": 7292, + "end_line": 7296, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_*foo*_\n", + "html": "

foo

\n", + "example": 462, + "start_line": 7299, + "end_line": 7303, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "****foo****\n", + "html": "

foo

\n", + "example": 463, + "start_line": 7309, + "end_line": 7313, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "____foo____\n", + "html": "

foo

\n", + "example": 464, + "start_line": 7316, + "end_line": 7320, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "******foo******\n", + "html": "

foo

\n", + "example": 465, + "start_line": 7327, + "end_line": 7331, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "***foo***\n", + "html": "

foo

\n", + "example": 466, + "start_line": 7336, + "end_line": 7340, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_____foo_____\n", + "html": "

foo

\n", + "example": 467, + "start_line": 7343, + "end_line": 7347, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo _bar* baz_\n", + "html": "

foo _bar baz_

\n", + "example": 468, + "start_line": 7352, + "end_line": 7356, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo __bar *baz bim__ bam*\n", + "html": "

foo bar *baz bim bam

\n", + "example": 469, + "start_line": 7359, + "end_line": 7363, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**foo **bar baz**\n", + "html": "

**foo bar baz

\n", + "example": 470, + "start_line": 7368, + "end_line": 7372, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*foo *bar baz*\n", + "html": "

*foo bar baz

\n", + "example": 471, + "start_line": 7375, + "end_line": 7379, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*[bar*](/url)\n", + "html": "

*bar*

\n", + "example": 472, + "start_line": 7384, + "end_line": 7388, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_foo [bar_](/url)\n", + "html": "

_foo bar_

\n", + "example": 473, + "start_line": 7391, + "end_line": 7395, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*\n", + "html": "

*

\n", + "example": 474, + "start_line": 7398, + "end_line": 7402, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**\n", + "html": "

**

\n", + "example": 475, + "start_line": 7405, + "end_line": 7409, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__\n", + "html": "

__

\n", + "example": 476, + "start_line": 7412, + "end_line": 7416, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "*a `*`*\n", + "html": "

a *

\n", + "example": 477, + "start_line": 7419, + "end_line": 7423, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "_a `_`_\n", + "html": "

a _

\n", + "example": 478, + "start_line": 7426, + "end_line": 7430, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "**a\n", + "html": "

**ahttp://foo.bar/?q=**

\n", + "example": 479, + "start_line": 7433, + "end_line": 7437, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "__a\n", + "html": "

__ahttp://foo.bar/?q=__

\n", + "example": 480, + "start_line": 7440, + "end_line": 7444, + "section": "Emphasis and strong emphasis" + }, + { + "markdown": "[link](/uri \"title\")\n", + "html": "

link

\n", + "example": 481, + "start_line": 7528, + "end_line": 7532, + "section": "Links" + }, + { + "markdown": "[link](/uri)\n", + "html": "

link

\n", + "example": 482, + "start_line": 7538, + "end_line": 7542, + "section": "Links" + }, + { + "markdown": "[](./target.md)\n", + "html": "

\n", + "example": 483, + "start_line": 7544, + "end_line": 7548, + "section": "Links" + }, + { + "markdown": "[link]()\n", + "html": "

link

\n", + "example": 484, + "start_line": 7551, + "end_line": 7555, + "section": "Links" + }, + { + "markdown": "[link](<>)\n", + "html": "

link

\n", + "example": 485, + "start_line": 7558, + "end_line": 7562, + "section": "Links" + }, + { + "markdown": "[]()\n", + "html": "

\n", + "example": 486, + "start_line": 7565, + "end_line": 7569, + "section": "Links" + }, + { + "markdown": "[link](/my uri)\n", + "html": "

[link](/my uri)

\n", + "example": 487, + "start_line": 7574, + "end_line": 7578, + "section": "Links" + }, + { + "markdown": "[link]()\n", + "html": "

link

\n", + "example": 488, + "start_line": 7580, + "end_line": 7584, + "section": "Links" + }, + { + "markdown": "[link](foo\nbar)\n", + "html": "

[link](foo\nbar)

\n", + "example": 489, + "start_line": 7589, + "end_line": 7595, + "section": "Links" + }, + { + "markdown": "[link]()\n", + "html": "

[link]()

\n", + "example": 490, + "start_line": 7597, + "end_line": 7603, + "section": "Links" + }, + { + "markdown": "[a]()\n", + "html": "

a

\n", + "example": 491, + "start_line": 7608, + "end_line": 7612, + "section": "Links" + }, + { + "markdown": "[link]()\n", + "html": "

[link](<foo>)

\n", + "example": 492, + "start_line": 7616, + "end_line": 7620, + "section": "Links" + }, + { + "markdown": "[a](\n[a](c)\n", + "html": "

[a](<b)c\n[a](<b)c>\n[a](c)

\n", + "example": 493, + "start_line": 7625, + "end_line": 7633, + "section": "Links" + }, + { + "markdown": "[link](\\(foo\\))\n", + "html": "

link

\n", + "example": 494, + "start_line": 7637, + "end_line": 7641, + "section": "Links" + }, + { + "markdown": "[link](foo(and(bar)))\n", + "html": "

link

\n", + "example": 495, + "start_line": 7646, + "end_line": 7650, + "section": "Links" + }, + { + "markdown": "[link](foo(and(bar))\n", + "html": "

[link](foo(and(bar))

\n", + "example": 496, + "start_line": 7655, + "end_line": 7659, + "section": "Links" + }, + { + "markdown": "[link](foo\\(and\\(bar\\))\n", + "html": "

link

\n", + "example": 497, + "start_line": 7662, + "end_line": 7666, + "section": "Links" + }, + { + "markdown": "[link]()\n", + "html": "

link

\n", + "example": 498, + "start_line": 7669, + "end_line": 7673, + "section": "Links" + }, + { + "markdown": "[link](foo\\)\\:)\n", + "html": "

link

\n", + "example": 499, + "start_line": 7679, + "end_line": 7683, + "section": "Links" + }, + { + "markdown": "[link](#fragment)\n\n[link](http://example.com#fragment)\n\n[link](http://example.com?foo=3#frag)\n", + "html": "

link

\n

link

\n

link

\n", + "example": 500, + "start_line": 7688, + "end_line": 7698, + "section": "Links" + }, + { + "markdown": "[link](foo\\bar)\n", + "html": "

link

\n", + "example": 501, + "start_line": 7704, + "end_line": 7708, + "section": "Links" + }, + { + "markdown": "[link](foo%20bä)\n", + "html": "

link

\n", + "example": 502, + "start_line": 7720, + "end_line": 7724, + "section": "Links" + }, + { + "markdown": "[link](\"title\")\n", + "html": "

link

\n", + "example": 503, + "start_line": 7731, + "end_line": 7735, + "section": "Links" + }, + { + "markdown": "[link](/url \"title\")\n[link](/url 'title')\n[link](/url (title))\n", + "html": "

link\nlink\nlink

\n", + "example": 504, + "start_line": 7740, + "end_line": 7748, + "section": "Links" + }, + { + "markdown": "[link](/url \"title \\\""\")\n", + "html": "

link

\n", + "example": 505, + "start_line": 7754, + "end_line": 7758, + "section": "Links" + }, + { + "markdown": "[link](/url \"title\")\n", + "html": "

link

\n", + "example": 506, + "start_line": 7765, + "end_line": 7769, + "section": "Links" + }, + { + "markdown": "[link](/url \"title \"and\" title\")\n", + "html": "

[link](/url "title "and" title")

\n", + "example": 507, + "start_line": 7774, + "end_line": 7778, + "section": "Links" + }, + { + "markdown": "[link](/url 'title \"and\" title')\n", + "html": "

link

\n", + "example": 508, + "start_line": 7783, + "end_line": 7787, + "section": "Links" + }, + { + "markdown": "[link]( /uri\n \"title\" )\n", + "html": "

link

\n", + "example": 509, + "start_line": 7808, + "end_line": 7813, + "section": "Links" + }, + { + "markdown": "[link] (/uri)\n", + "html": "

[link] (/uri)

\n", + "example": 510, + "start_line": 7819, + "end_line": 7823, + "section": "Links" + }, + { + "markdown": "[link [foo [bar]]](/uri)\n", + "html": "

link [foo [bar]]

\n", + "example": 511, + "start_line": 7829, + "end_line": 7833, + "section": "Links" + }, + { + "markdown": "[link] bar](/uri)\n", + "html": "

[link] bar](/uri)

\n", + "example": 512, + "start_line": 7836, + "end_line": 7840, + "section": "Links" + }, + { + "markdown": "[link [bar](/uri)\n", + "html": "

[link bar

\n", + "example": 513, + "start_line": 7843, + "end_line": 7847, + "section": "Links" + }, + { + "markdown": "[link \\[bar](/uri)\n", + "html": "

link [bar

\n", + "example": 514, + "start_line": 7850, + "end_line": 7854, + "section": "Links" + }, + { + "markdown": "[link *foo **bar** `#`*](/uri)\n", + "html": "

link foo bar #

\n", + "example": 515, + "start_line": 7859, + "end_line": 7863, + "section": "Links" + }, + { + "markdown": "[![moon](moon.jpg)](/uri)\n", + "html": "

\"moon\"

\n", + "example": 516, + "start_line": 7866, + "end_line": 7870, + "section": "Links" + }, + { + "markdown": "[foo [bar](/uri)](/uri)\n", + "html": "

[foo bar](/uri)

\n", + "example": 517, + "start_line": 7875, + "end_line": 7879, + "section": "Links" + }, + { + "markdown": "[foo *[bar [baz](/uri)](/uri)*](/uri)\n", + "html": "

[foo [bar baz](/uri)](/uri)

\n", + "example": 518, + "start_line": 7882, + "end_line": 7886, + "section": "Links" + }, + { + "markdown": "![[[foo](uri1)](uri2)](uri3)\n", + "html": "

\"[foo](uri2)\"

\n", + "example": 519, + "start_line": 7889, + "end_line": 7893, + "section": "Links" + }, + { + "markdown": "*[foo*](/uri)\n", + "html": "

*foo*

\n", + "example": 520, + "start_line": 7899, + "end_line": 7903, + "section": "Links" + }, + { + "markdown": "[foo *bar](baz*)\n", + "html": "

foo *bar

\n", + "example": 521, + "start_line": 7906, + "end_line": 7910, + "section": "Links" + }, + { + "markdown": "*foo [bar* baz]\n", + "html": "

foo [bar baz]

\n", + "example": 522, + "start_line": 7916, + "end_line": 7920, + "section": "Links" + }, + { + "markdown": "[foo \n", + "html": "

[foo

\n", + "example": 523, + "start_line": 7926, + "end_line": 7930, + "section": "Links" + }, + { + "markdown": "[foo`](/uri)`\n", + "html": "

[foo](/uri)

\n", + "example": 524, + "start_line": 7933, + "end_line": 7937, + "section": "Links" + }, + { + "markdown": "[foo\n", + "html": "

[foohttp://example.com/?search=](uri)

\n", + "example": 525, + "start_line": 7940, + "end_line": 7944, + "section": "Links" + }, + { + "markdown": "[foo][bar]\n\n[bar]: /url \"title\"\n", + "html": "

foo

\n", + "example": 526, + "start_line": 7978, + "end_line": 7984, + "section": "Links" + }, + { + "markdown": "[link [foo [bar]]][ref]\n\n[ref]: /uri\n", + "html": "

link [foo [bar]]

\n", + "example": 527, + "start_line": 7993, + "end_line": 7999, + "section": "Links" + }, + { + "markdown": "[link \\[bar][ref]\n\n[ref]: /uri\n", + "html": "

link [bar

\n", + "example": 528, + "start_line": 8002, + "end_line": 8008, + "section": "Links" + }, + { + "markdown": "[link *foo **bar** `#`*][ref]\n\n[ref]: /uri\n", + "html": "

link foo bar #

\n", + "example": 529, + "start_line": 8013, + "end_line": 8019, + "section": "Links" + }, + { + "markdown": "[![moon](moon.jpg)][ref]\n\n[ref]: /uri\n", + "html": "

\"moon\"

\n", + "example": 530, + "start_line": 8022, + "end_line": 8028, + "section": "Links" + }, + { + "markdown": "[foo [bar](/uri)][ref]\n\n[ref]: /uri\n", + "html": "

[foo bar]ref

\n", + "example": 531, + "start_line": 8033, + "end_line": 8039, + "section": "Links" + }, + { + "markdown": "[foo *bar [baz][ref]*][ref]\n\n[ref]: /uri\n", + "html": "

[foo bar baz]ref

\n", + "example": 532, + "start_line": 8042, + "end_line": 8048, + "section": "Links" + }, + { + "markdown": "*[foo*][ref]\n\n[ref]: /uri\n", + "html": "

*foo*

\n", + "example": 533, + "start_line": 8057, + "end_line": 8063, + "section": "Links" + }, + { + "markdown": "[foo *bar][ref]*\n\n[ref]: /uri\n", + "html": "

foo *bar*

\n", + "example": 534, + "start_line": 8066, + "end_line": 8072, + "section": "Links" + }, + { + "markdown": "[foo \n\n[ref]: /uri\n", + "html": "

[foo

\n", + "example": 535, + "start_line": 8078, + "end_line": 8084, + "section": "Links" + }, + { + "markdown": "[foo`][ref]`\n\n[ref]: /uri\n", + "html": "

[foo][ref]

\n", + "example": 536, + "start_line": 8087, + "end_line": 8093, + "section": "Links" + }, + { + "markdown": "[foo\n\n[ref]: /uri\n", + "html": "

[foohttp://example.com/?search=][ref]

\n", + "example": 537, + "start_line": 8096, + "end_line": 8102, + "section": "Links" + }, + { + "markdown": "[foo][BaR]\n\n[bar]: /url \"title\"\n", + "html": "

foo

\n", + "example": 538, + "start_line": 8107, + "end_line": 8113, + "section": "Links" + }, + { + "markdown": "[ẞ]\n\n[SS]: /url\n", + "html": "

ẞ

\n", + "example": 539, + "start_line": 8118, + "end_line": 8124, + "section": "Links" + }, + { + "markdown": "[Foo\n bar]: /url\n\n[Baz][Foo bar]\n", + "html": "

Baz

\n", + "example": 540, + "start_line": 8130, + "end_line": 8137, + "section": "Links" + }, + { + "markdown": "[foo] [bar]\n\n[bar]: /url \"title\"\n", + "html": "

[foo] bar

\n", + "example": 541, + "start_line": 8143, + "end_line": 8149, + "section": "Links" + }, + { + "markdown": "[foo]\n[bar]\n\n[bar]: /url \"title\"\n", + "html": "

[foo]\nbar

\n", + "example": 542, + "start_line": 8152, + "end_line": 8160, + "section": "Links" + }, + { + "markdown": "[foo]: /url1\n\n[foo]: /url2\n\n[bar][foo]\n", + "html": "

bar

\n", + "example": 543, + "start_line": 8193, + "end_line": 8201, + "section": "Links" + }, + { + "markdown": "[bar][foo\\!]\n\n[foo!]: /url\n", + "html": "

[bar][foo!]

\n", + "example": 544, + "start_line": 8208, + "end_line": 8214, + "section": "Links" + }, + { + "markdown": "[foo][ref[]\n\n[ref[]: /uri\n", + "html": "

[foo][ref[]

\n

[ref[]: /uri

\n", + "example": 545, + "start_line": 8220, + "end_line": 8227, + "section": "Links" + }, + { + "markdown": "[foo][ref[bar]]\n\n[ref[bar]]: /uri\n", + "html": "

[foo][ref[bar]]

\n

[ref[bar]]: /uri

\n", + "example": 546, + "start_line": 8230, + "end_line": 8237, + "section": "Links" + }, + { + "markdown": "[[[foo]]]\n\n[[[foo]]]: /url\n", + "html": "

[[[foo]]]

\n

[[[foo]]]: /url

\n", + "example": 547, + "start_line": 8240, + "end_line": 8247, + "section": "Links" + }, + { + "markdown": "[foo][ref\\[]\n\n[ref\\[]: /uri\n", + "html": "

foo

\n", + "example": 548, + "start_line": 8250, + "end_line": 8256, + "section": "Links" + }, + { + "markdown": "[bar\\\\]: /uri\n\n[bar\\\\]\n", + "html": "

bar\\

\n", + "example": 549, + "start_line": 8261, + "end_line": 8267, + "section": "Links" + }, + { + "markdown": "[]\n\n[]: /uri\n", + "html": "

[]

\n

[]: /uri

\n", + "example": 550, + "start_line": 8273, + "end_line": 8280, + "section": "Links" + }, + { + "markdown": "[\n ]\n\n[\n ]: /uri\n", + "html": "

[\n]

\n

[\n]: /uri

\n", + "example": 551, + "start_line": 8283, + "end_line": 8294, + "section": "Links" + }, + { + "markdown": "[foo][]\n\n[foo]: /url \"title\"\n", + "html": "

foo

\n", + "example": 552, + "start_line": 8306, + "end_line": 8312, + "section": "Links" + }, + { + "markdown": "[*foo* bar][]\n\n[*foo* bar]: /url \"title\"\n", + "html": "

foo bar

\n", + "example": 553, + "start_line": 8315, + "end_line": 8321, + "section": "Links" + }, + { + "markdown": "[Foo][]\n\n[foo]: /url \"title\"\n", + "html": "

Foo

\n", + "example": 554, + "start_line": 8326, + "end_line": 8332, + "section": "Links" + }, + { + "markdown": "[foo] \n[]\n\n[foo]: /url \"title\"\n", + "html": "

foo\n[]

\n", + "example": 555, + "start_line": 8339, + "end_line": 8347, + "section": "Links" + }, + { + "markdown": "[foo]\n\n[foo]: /url \"title\"\n", + "html": "

foo

\n", + "example": 556, + "start_line": 8359, + "end_line": 8365, + "section": "Links" + }, + { + "markdown": "[*foo* bar]\n\n[*foo* bar]: /url \"title\"\n", + "html": "

foo bar

\n", + "example": 557, + "start_line": 8368, + "end_line": 8374, + "section": "Links" + }, + { + "markdown": "[[*foo* bar]]\n\n[*foo* bar]: /url \"title\"\n", + "html": "

[foo bar]

\n", + "example": 558, + "start_line": 8377, + "end_line": 8383, + "section": "Links" + }, + { + "markdown": "[[bar [foo]\n\n[foo]: /url\n", + "html": "

[[bar foo

\n", + "example": 559, + "start_line": 8386, + "end_line": 8392, + "section": "Links" + }, + { + "markdown": "[Foo]\n\n[foo]: /url \"title\"\n", + "html": "

Foo

\n", + "example": 560, + "start_line": 8397, + "end_line": 8403, + "section": "Links" + }, + { + "markdown": "[foo] bar\n\n[foo]: /url\n", + "html": "

foo bar

\n", + "example": 561, + "start_line": 8408, + "end_line": 8414, + "section": "Links" + }, + { + "markdown": "\\[foo]\n\n[foo]: /url \"title\"\n", + "html": "

[foo]

\n", + "example": 562, + "start_line": 8420, + "end_line": 8426, + "section": "Links" + }, + { + "markdown": "[foo*]: /url\n\n*[foo*]\n", + "html": "

*foo*

\n", + "example": 563, + "start_line": 8432, + "end_line": 8438, + "section": "Links" + }, + { + "markdown": "[foo][bar]\n\n[foo]: /url1\n[bar]: /url2\n", + "html": "

foo

\n", + "example": 564, + "start_line": 8444, + "end_line": 8451, + "section": "Links" + }, + { + "markdown": "[foo][]\n\n[foo]: /url1\n", + "html": "

foo

\n", + "example": 565, + "start_line": 8453, + "end_line": 8459, + "section": "Links" + }, + { + "markdown": "[foo]()\n\n[foo]: /url1\n", + "html": "

foo

\n", + "example": 566, + "start_line": 8463, + "end_line": 8469, + "section": "Links" + }, + { + "markdown": "[foo](not a link)\n\n[foo]: /url1\n", + "html": "

foo(not a link)

\n", + "example": 567, + "start_line": 8471, + "end_line": 8477, + "section": "Links" + }, + { + "markdown": "[foo][bar][baz]\n\n[baz]: /url\n", + "html": "

[foo]bar

\n", + "example": 568, + "start_line": 8482, + "end_line": 8488, + "section": "Links" + }, + { + "markdown": "[foo][bar][baz]\n\n[baz]: /url1\n[bar]: /url2\n", + "html": "

foobaz

\n", + "example": 569, + "start_line": 8494, + "end_line": 8501, + "section": "Links" + }, + { + "markdown": "[foo][bar][baz]\n\n[baz]: /url1\n[foo]: /url2\n", + "html": "

[foo]bar

\n", + "example": 570, + "start_line": 8507, + "end_line": 8514, + "section": "Links" + }, + { + "markdown": "![foo](/url \"title\")\n", + "html": "

\"foo\"

\n", + "example": 571, + "start_line": 8530, + "end_line": 8534, + "section": "Images" + }, + { + "markdown": "![foo *bar*]\n\n[foo *bar*]: train.jpg \"train & tracks\"\n", + "html": "

\"foo

\n", + "example": 572, + "start_line": 8537, + "end_line": 8543, + "section": "Images" + }, + { + "markdown": "![foo ![bar](/url)](/url2)\n", + "html": "

\"foo

\n", + "example": 573, + "start_line": 8546, + "end_line": 8550, + "section": "Images" + }, + { + "markdown": "![foo [bar](/url)](/url2)\n", + "html": "

\"foo

\n", + "example": 574, + "start_line": 8553, + "end_line": 8557, + "section": "Images" + }, + { + "markdown": "![foo *bar*][]\n\n[foo *bar*]: train.jpg \"train & tracks\"\n", + "html": "

\"foo

\n", + "example": 575, + "start_line": 8567, + "end_line": 8573, + "section": "Images" + }, + { + "markdown": "![foo *bar*][foobar]\n\n[FOOBAR]: train.jpg \"train & tracks\"\n", + "html": "

\"foo

\n", + "example": 576, + "start_line": 8576, + "end_line": 8582, + "section": "Images" + }, + { + "markdown": "![foo](train.jpg)\n", + "html": "

\"foo\"

\n", + "example": 577, + "start_line": 8585, + "end_line": 8589, + "section": "Images" + }, + { + "markdown": "My ![foo bar](/path/to/train.jpg \"title\" )\n", + "html": "

My \"foo

\n", + "example": 578, + "start_line": 8592, + "end_line": 8596, + "section": "Images" + }, + { + "markdown": "![foo]()\n", + "html": "

\"foo\"

\n", + "example": 579, + "start_line": 8599, + "end_line": 8603, + "section": "Images" + }, + { + "markdown": "![](/url)\n", + "html": "

\"\"

\n", + "example": 580, + "start_line": 8606, + "end_line": 8610, + "section": "Images" + }, + { + "markdown": "![foo][bar]\n\n[bar]: /url\n", + "html": "

\"foo\"

\n", + "example": 581, + "start_line": 8615, + "end_line": 8621, + "section": "Images" + }, + { + "markdown": "![foo][bar]\n\n[BAR]: /url\n", + "html": "

\"foo\"

\n", + "example": 582, + "start_line": 8624, + "end_line": 8630, + "section": "Images" + }, + { + "markdown": "![foo][]\n\n[foo]: /url \"title\"\n", + "html": "

\"foo\"

\n", + "example": 583, + "start_line": 8635, + "end_line": 8641, + "section": "Images" + }, + { + "markdown": "![*foo* bar][]\n\n[*foo* bar]: /url \"title\"\n", + "html": "

\"foo

\n", + "example": 584, + "start_line": 8644, + "end_line": 8650, + "section": "Images" + }, + { + "markdown": "![Foo][]\n\n[foo]: /url \"title\"\n", + "html": "

\"Foo\"

\n", + "example": 585, + "start_line": 8655, + "end_line": 8661, + "section": "Images" + }, + { + "markdown": "![foo] \n[]\n\n[foo]: /url \"title\"\n", + "html": "

\"foo\"\n[]

\n", + "example": 586, + "start_line": 8667, + "end_line": 8675, + "section": "Images" + }, + { + "markdown": "![foo]\n\n[foo]: /url \"title\"\n", + "html": "

\"foo\"

\n", + "example": 587, + "start_line": 8680, + "end_line": 8686, + "section": "Images" + }, + { + "markdown": "![*foo* bar]\n\n[*foo* bar]: /url \"title\"\n", + "html": "

\"foo

\n", + "example": 588, + "start_line": 8689, + "end_line": 8695, + "section": "Images" + }, + { + "markdown": "![[foo]]\n\n[[foo]]: /url \"title\"\n", + "html": "

![[foo]]

\n

[[foo]]: /url "title"

\n", + "example": 589, + "start_line": 8700, + "end_line": 8707, + "section": "Images" + }, + { + "markdown": "![Foo]\n\n[foo]: /url \"title\"\n", + "html": "

\"Foo\"

\n", + "example": 590, + "start_line": 8712, + "end_line": 8718, + "section": "Images" + }, + { + "markdown": "!\\[foo]\n\n[foo]: /url \"title\"\n", + "html": "

![foo]

\n", + "example": 591, + "start_line": 8724, + "end_line": 8730, + "section": "Images" + }, + { + "markdown": "\\![foo]\n\n[foo]: /url \"title\"\n", + "html": "

!foo

\n", + "example": 592, + "start_line": 8736, + "end_line": 8742, + "section": "Images" + }, + { + "markdown": "\n", + "html": "

http://foo.bar.baz

\n", + "example": 593, + "start_line": 8769, + "end_line": 8773, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

http://foo.bar.baz/test?q=hello&id=22&boolean

\n", + "example": 594, + "start_line": 8776, + "end_line": 8780, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

irc://foo.bar:2233/baz

\n", + "example": 595, + "start_line": 8783, + "end_line": 8787, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

MAILTO:FOO@BAR.BAZ

\n", + "example": 596, + "start_line": 8792, + "end_line": 8796, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

a+b+c:d

\n", + "example": 597, + "start_line": 8804, + "end_line": 8808, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

made-up-scheme://foo,bar

\n", + "example": 598, + "start_line": 8811, + "end_line": 8815, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

http://../

\n", + "example": 599, + "start_line": 8818, + "end_line": 8822, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

localhost:5001/foo

\n", + "example": 600, + "start_line": 8825, + "end_line": 8829, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

<http://foo.bar/baz bim>

\n", + "example": 601, + "start_line": 8834, + "end_line": 8838, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

http://example.com/\\[\\

\n", + "example": 602, + "start_line": 8843, + "end_line": 8847, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

foo@bar.example.com

\n", + "example": 603, + "start_line": 8865, + "end_line": 8869, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

foo+special@Bar.baz-bar0.com

\n", + "example": 604, + "start_line": 8872, + "end_line": 8876, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

<foo+@bar.example.com>

\n", + "example": 605, + "start_line": 8881, + "end_line": 8885, + "section": "Autolinks" + }, + { + "markdown": "<>\n", + "html": "

<>

\n", + "example": 606, + "start_line": 8890, + "end_line": 8894, + "section": "Autolinks" + }, + { + "markdown": "< http://foo.bar >\n", + "html": "

< http://foo.bar >

\n", + "example": 607, + "start_line": 8897, + "end_line": 8901, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

<m:abc>

\n", + "example": 608, + "start_line": 8904, + "end_line": 8908, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

<foo.bar.baz>

\n", + "example": 609, + "start_line": 8911, + "end_line": 8915, + "section": "Autolinks" + }, + { + "markdown": "http://example.com\n", + "html": "

http://example.com

\n", + "example": 610, + "start_line": 8918, + "end_line": 8922, + "section": "Autolinks" + }, + { + "markdown": "foo@bar.example.com\n", + "html": "

foo@bar.example.com

\n", + "example": 611, + "start_line": 8925, + "end_line": 8929, + "section": "Autolinks" + }, + { + "markdown": "\n", + "html": "

\n", + "example": 612, + "start_line": 9006, + "end_line": 9010, + "section": "Raw HTML" + }, + { + "markdown": "\n", + "html": "

\n", + "example": 613, + "start_line": 9015, + "end_line": 9019, + "section": "Raw HTML" + }, + { + "markdown": "\n", + "html": "

\n", + "example": 614, + "start_line": 9024, + "end_line": 9030, + "section": "Raw HTML" + }, + { + "markdown": "\n", + "html": "

\n", + "example": 615, + "start_line": 9035, + "end_line": 9041, + "section": "Raw HTML" + }, + { + "markdown": "Foo \n", + "html": "

Foo

\n", + "example": 616, + "start_line": 9046, + "end_line": 9050, + "section": "Raw HTML" + }, + { + "markdown": "<33> <__>\n", + "html": "

<33> <__>

\n", + "example": 617, + "start_line": 9055, + "end_line": 9059, + "section": "Raw HTML" + }, + { + "markdown": "\n", + "html": "

<a h*#ref="hi">

\n", + "example": 618, + "start_line": 9064, + "end_line": 9068, + "section": "Raw HTML" + }, + { + "markdown": " \n", + "html": "

<a href="hi'> <a href=hi'>

\n", + "example": 619, + "start_line": 9073, + "end_line": 9077, + "section": "Raw HTML" + }, + { + "markdown": "< a><\nfoo>\n\n", + "html": "

< a><\nfoo><bar/ >\n<foo bar=baz\nbim!bop />

\n", + "example": 620, + "start_line": 9082, + "end_line": 9092, + "section": "Raw HTML" + }, + { + "markdown": "\n", + "html": "

<a href='bar'title=title>

\n", + "example": 621, + "start_line": 9097, + "end_line": 9101, + "section": "Raw HTML" + }, + { + "markdown": "\n", + "html": "

\n", + "example": 622, + "start_line": 9106, + "end_line": 9110, + "section": "Raw HTML" + }, + { + "markdown": "\n", + "html": "

</a href="foo">

\n", + "example": 623, + "start_line": 9115, + "end_line": 9119, + "section": "Raw HTML" + }, + { + "markdown": "foo \n", + "html": "

foo

\n", + "example": 624, + "start_line": 9124, + "end_line": 9130, + "section": "Raw HTML" + }, + { + "markdown": "foo \n", + "html": "

foo <!-- not a comment -- two hyphens -->

\n", + "example": 625, + "start_line": 9133, + "end_line": 9137, + "section": "Raw HTML" + }, + { + "markdown": "foo foo -->\n\nfoo \n", + "html": "

foo <!--> foo -->

\n

foo <!-- foo--->

\n", + "example": 626, + "start_line": 9142, + "end_line": 9149, + "section": "Raw HTML" + }, + { + "markdown": "foo \n", + "html": "

foo

\n", + "example": 627, + "start_line": 9154, + "end_line": 9158, + "section": "Raw HTML" + }, + { + "markdown": "foo \n", + "html": "

foo

\n", + "example": 628, + "start_line": 9163, + "end_line": 9167, + "section": "Raw HTML" + }, + { + "markdown": "foo &<]]>\n", + "html": "

foo &<]]>

\n", + "example": 629, + "start_line": 9172, + "end_line": 9176, + "section": "Raw HTML" + }, + { + "markdown": "foo \n", + "html": "

foo

\n", + "example": 630, + "start_line": 9182, + "end_line": 9186, + "section": "Raw HTML" + }, + { + "markdown": "foo \n", + "html": "

foo

\n", + "example": 631, + "start_line": 9191, + "end_line": 9195, + "section": "Raw HTML" + }, + { + "markdown": "\n", + "html": "

<a href=""">

\n", + "example": 632, + "start_line": 9198, + "end_line": 9202, + "section": "Raw HTML" + }, + { + "markdown": "foo \nbaz\n", + "html": "

foo
\nbaz

\n", + "example": 633, + "start_line": 9212, + "end_line": 9218, + "section": "Hard line breaks" + }, + { + "markdown": "foo\\\nbaz\n", + "html": "

foo
\nbaz

\n", + "example": 634, + "start_line": 9224, + "end_line": 9230, + "section": "Hard line breaks" + }, + { + "markdown": "foo \nbaz\n", + "html": "

foo
\nbaz

\n", + "example": 635, + "start_line": 9235, + "end_line": 9241, + "section": "Hard line breaks" + }, + { + "markdown": "foo \n bar\n", + "html": "

foo
\nbar

\n", + "example": 636, + "start_line": 9246, + "end_line": 9252, + "section": "Hard line breaks" + }, + { + "markdown": "foo\\\n bar\n", + "html": "

foo
\nbar

\n", + "example": 637, + "start_line": 9255, + "end_line": 9261, + "section": "Hard line breaks" + }, + { + "markdown": "*foo \nbar*\n", + "html": "

foo
\nbar

\n", + "example": 638, + "start_line": 9267, + "end_line": 9273, + "section": "Hard line breaks" + }, + { + "markdown": "*foo\\\nbar*\n", + "html": "

foo
\nbar

\n", + "example": 639, + "start_line": 9276, + "end_line": 9282, + "section": "Hard line breaks" + }, + { + "markdown": "`code \nspan`\n", + "html": "

code span

\n", + "example": 640, + "start_line": 9287, + "end_line": 9292, + "section": "Hard line breaks" + }, + { + "markdown": "`code\\\nspan`\n", + "html": "

code\\ span

\n", + "example": 641, + "start_line": 9295, + "end_line": 9300, + "section": "Hard line breaks" + }, + { + "markdown": "\n", + "html": "

\n", + "example": 642, + "start_line": 9305, + "end_line": 9311, + "section": "Hard line breaks" + }, + { + "markdown": "\n", + "html": "

\n", + "example": 643, + "start_line": 9314, + "end_line": 9320, + "section": "Hard line breaks" + }, + { + "markdown": "foo\\\n", + "html": "

foo\\

\n", + "example": 644, + "start_line": 9327, + "end_line": 9331, + "section": "Hard line breaks" + }, + { + "markdown": "foo \n", + "html": "

foo

\n", + "example": 645, + "start_line": 9334, + "end_line": 9338, + "section": "Hard line breaks" + }, + { + "markdown": "### foo\\\n", + "html": "

foo\\

\n", + "example": 646, + "start_line": 9341, + "end_line": 9345, + "section": "Hard line breaks" + }, + { + "markdown": "### foo \n", + "html": "

foo

\n", + "example": 647, + "start_line": 9348, + "end_line": 9352, + "section": "Hard line breaks" + }, + { + "markdown": "foo\nbaz\n", + "html": "

foo\nbaz

\n", + "example": 648, + "start_line": 9363, + "end_line": 9369, + "section": "Soft line breaks" + }, + { + "markdown": "foo \n baz\n", + "html": "

foo\nbaz

\n", + "example": 649, + "start_line": 9375, + "end_line": 9381, + "section": "Soft line breaks" + }, + { + "markdown": "hello $.;'there\n", + "html": "

hello $.;'there

\n", + "example": 650, + "start_line": 9395, + "end_line": 9399, + "section": "Textual content" + }, + { + "markdown": "Foo χρῆν\n", + "html": "

Foo χρῆν

\n", + "example": 651, + "start_line": 9402, + "end_line": 9406, + "section": "Textual content" + }, + { + "markdown": "Multiple spaces\n", + "html": "

Multiple spaces

\n", + "example": 652, + "start_line": 9411, + "end_line": 9415, + "section": "Textual content" + } +] \ No newline at end of file diff --git a/pkg/markdown/spec/gfm_v.0.29.json b/pkg/markdown/spec/gfm_v.0.29.json new file mode 100644 index 00000000..59e3d732 --- /dev/null +++ b/pkg/markdown/spec/gfm_v.0.29.json @@ -0,0 +1,148 @@ +[ + { + "markdown": "| foo | bar |\n| --- | --- |\n| baz | bim |\n", + "html": "\n\n\n\n\n\n\n\n\n\n\n\n\n
foobar
bazbim
\n", + "example": 198, + "section": "Tables" + }, + { + "markdown": "| abc | defghi |\n:-: | -----------:\nbar | baz\n", + "html": "\n\n\n\n\n\n\n\n\n\n\n\n\n
abcdefghi
barbaz
\n", + "example": 199, + "section": "Tables" + }, + { + "markdown": "| f\\|oo |\n| ------ |\n| b `\\|` az |\n| b **\\|** im |\n", + "html": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n
f|oo
b | az
b | im
\n", + "example": 200, + "section": "Tables" + }, + { + "markdown": "| abc | def |\n| --- | --- |\n| bar | baz |\n> bar\n", + "html": "\n\n\n\n\n\n\n\n\n\n\n\n\n
abcdef
barbaz
\n
\n

bar

\n
\n", + "example": 201, + "section": "Tables" + }, + { + "markdown": "| abc | def |\n| --- | --- |\n| bar | baz |\nbar\n\nbar\n", + "html": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
abcdef
barbaz
bar
\n

bar

", + "example": 202, + "section": "Tables" + }, + { + "markdown": "| abc | def |\n| --- | --- |\n| bar | baz |\n> bar\n", + "html": "

| abc | def |\n| --- |\n| bar |

\n", + "example": 203, + "section": "Tables" + }, + { + "markdown": "| abc | def |\n| --- | --- |\n| bar |\n| bar | baz | boo |\n", + "html": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
abcdef
bar
barbaz
\n", + "example": 204, + "section": "Tables" + }, + { + "markdown": "| abc | def |\n| --- | --- |\n", + "html": "\n\n\n\n\n\n\n
abcdef
", + "example": 205, + "section": "Tables" + }, + { + "markdown": "- [ ] foo\n- [x] bar\n", + "html": "
    \n
  • foo
    • \n
  • bar
    • \n
\n", + "example": 279, + "section": "Task list items" + }, + { + "markdown": "- [x] foo\n - [ ] bar\n - [x] baz\n- [ ] bim\n", + "html": "
    \n
  • foo\n
      \n
    • bar
      • \n
    • baz
      • \n
    \n
    • \n
  • bim
    • \n
\n", + "example": 280, + "section": "Task list items" + }, + { + "markdown": "~~Hi~~ Hello, world!\n", + "html": "

Hi Hello, world!

\n", + "example": 491, + "section": "Strikethrough" + }, + { + "markdown": "This ~~has a\n\nnew paragraph~~.\n", + "html": "

This ~~has a

\n

new paragraph~~.

\n", + "example": 492, + "section": "Strikethrough" + }, + { + "markdown": "www.commonmark.org\n", + "html": "

www.commonmark.org

\n", + "example": 621, + "section": "Autolinks" + }, + { + "markdown": "www.commonmark.org\n", + "html": "

www.commonmark.org

\n", + "example": 622, + "section": "Autolinks" + }, + { + "markdown": "Visit www.commonmark.org.\n\nVisit www.commonmark.org/a.b.\n", + "html": "

Visit www.commonmark.org.

\n

Visit www.commonmark.org/a.b.

\n", + "example": 623, + "section": "Autolinks" + }, + { + "markdown": "www.google.com/search?q=Markup+(business)\n\nwww.google.com/search?q=Markup+(business)))\n\n(www.google.com/search?q=Markup+(business))\n\n(www.google.com/search?q=Markup+(business)\n", + "html": "

www.google.com/search?q=Markup+(business)

\n

www.google.com/search?q=Markup+(business)))

\n

(www.google.com/search?q=Markup+(business))

\n

(www.google.com/search?q=Markup+(business)

\n", + "example": 624, + "section": "Autolinks" + }, + { + "markdown": "www.google.com/search?q=(business))+ok\n", + "html": "

www.google.com/search?q=(business))+ok

\n", + "example": 625, + "section": "Autolinks" + }, + { + "markdown": "www.google.com/search?q=commonmark&hl=en\n\nwww.google.com/search?q=commonmark&hl;\n", + "html": "

www.google.com/search?q=commonmark&hl=en

\n

www.google.com/search?q=commonmark&hl;

\n", + "example": 626, + "section": "Autolinks" + }, + { + "markdown": "www.commonmark.org/hewww.commonmark.org/he<lp

\n", + "example": 627, + "section": "Autolinks" + }, + { + "markdown": "http://commonmark.org\n\n(Visit https://encrypted.google.com/search?q=Markup+(business))\n", + "html": "

http://commonmark.org

\n

(Visit https://encrypted.google.com/search?q=Markup+(business))

\n", + "example": 628, + "section": "Autolinks" + }, + { + "markdown": "foo@bar.baz\n", + "html": "

foo@bar.baz

\n", + "example": 629, + "section": "Autolinks" + }, + { + "markdown": "hello@mail+xyz.example isn't valid, but hello+xyz@mail.example is.\n", + "html": "

hello@mail+xyz.example isn't valid, but hello+xyz@mail.example is.

\n", + "example": 630, + "section": "Autolinks" + }, + { + "markdown": "a.b-c_d@a.b\n\na.b-c_d@a.b.\n\na.b-c_d@a.b-\n\na.b-c_d@a.b_\n", + "html": "

a.b-c_d@a.b

\n

a.b-c_d@a.b.

\n

a.b-c_d@a.b-

\n

a.b-c_d@a.b_

\n", + "example": 631, + "section": "Autolinks" + }, + { + "markdown": " <style> <em>\n\n<blockquote>\n <xmp> is disallowed. <XMP> is also disallowed.\n</blockquote>\n", + "html": "<p><strong> <title> <style> <em></p>\n<blockquote>\n <xmp> is disallowed. <XMP> is also disallowed.\n</blockquote>\n", + "example": 653, + "section": "Disallowed Raw HTML" + } +] + + diff --git a/pkg/markdown/specs_test.go b/pkg/markdown/specs_test.go new file mode 100644 index 00000000..3f5ab476 --- /dev/null +++ b/pkg/markdown/specs_test.go @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package markdown_test + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/gardener/docforge/pkg/markdown" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension" + gmhtml "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "golang.org/x/net/html" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" +) + +// Goldmark Markdown with GFM extensions +var gm = goldmark.New(goldmark.WithRendererOptions(gmhtml.WithUnsafe()), goldmark.WithExtensions(extension.GFM)) + +// Link modifier renderer +var lmr = markdown.NewLinkModifierRenderer() + +type spec struct { + Markdown string `json:"markdown"` + HTML string `json:"html"` + Example int `json:"example"` +} + +type specs []spec + +// TestCommonmark executes commonmark compliance tests +func TestCommonmark(t *testing.T) { + testSpecs, err := loadSpec("commonmark_v.0.30.json") + if err != nil { + t.Errorf("error load specs %v", err) + } + for _, tc := range testSpecs { + t.Run(fmt.Sprintf("example %d", tc.Example), tc.executeSpecTest) + } +} + +func TestGFM(t *testing.T) { + testSpecs, err := loadSpec("gfm_v.0.29.json") + if err != nil { + t.Errorf("error load specs %v", err) + } + for _, tc := range testSpecs { + t.Run(fmt.Sprintf("example %d", tc.Example), tc.executeSpecTest) + } +} + +func TestCMarkGFMAfl(t *testing.T) { + tc := &spec{ + Markdown: "# H1\n\nH2\n--\n\nt ☺ \n*b* **em** `c`\n≥\\&\\\n\\_e\\_\n\n4) I1\n\n5) I2\n > [l](/u \"t\")\n >\n > - [f]\n > - ![a](/u \"t\")\n >\n >> <ftp://hh>\n >> <u@hh>\n\n~~~ l☺\ncb\n~~~\n\n c1\n c2\n\n***\n\n<div>\n<b>x</b>\n</div>\n\n| a | b |\n| --- | --- |\n| c | `d|` \\| e |\n\ngoogle ~~yahoo~~\n\ngoogle.com http://google.com google@google.com\n\nand <xmp> but\n\n<surewhynot>\nsure\n</surewhynot>\n\n[f]: /u \"t\"\n", + HTML: "<h1>H1</h1>\n<h2>H2</h2>\n<p>t ☺<br>\n<em>b</em> <strong>em</strong> <code>c</code>\n≥&<br>\n_e_</p>\n<ol start=\"4\">\n<li>\n<p>I1</p>\n</li>\n<li>\n<p>I2</p>\n<blockquote>\n<p><a href=\"/u\" title=\"t\">l</a></p>\n<ul>\n<li><a href=\"/u\" title=\"t\">f</a></li>\n<li><img src=\"/u\" alt=\"a\" title=\"t\"></li>\n</ul>\n<blockquote>\n<p><a href=\"ftp://hh\">ftp://hh</a>\n<a href=\"mailto:u@hh\">u@hh</a></p>\n</blockquote>\n</blockquote>\n</li>\n</ol>\n<pre><code class=\"language-l☺\">cb\n</code></pre>\n<pre><code>c1\nc2\n</code></pre>\n<hr>\n<div>\n<b>x</b>\n</div>\n<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>c</td>\n<td>`d</td>\n</tr>\n</tbody>\n</table>\n<p>google <del>yahoo</del></p>\n<p>google.com <a href=\"http://google.com\">http://google.com</a> <a href=\"mailto:google@google.com\">google@google.com</a></p>\n<p>and <xmp> but</p>\n<surewhynot>\nsure\n</surewhynot>\n", + Example: 1, + } + tc.executeSpecTest(t) +} + +func loadSpec(specName string) ([]spec, error) { + path, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("error getting working folder %v", err) + } + sep := string(filepath.Separator) + path = strings.TrimSuffix(path, sep) + // adjust path if test is not run from repo root + d, f := filepath.Split(path) + if f == "markdown" { + path = strings.TrimSuffix(d, sep) + d, f = filepath.Split(path) + } + if f == "pkg" { + path = strings.TrimSuffix(d, sep) + } + specFile := filepath.Join(path, "pkg", "markdown", "spec", specName) + var specBytes []byte + specBytes, err = ioutil.ReadFile(specFile) + if err != nil { + return nil, fmt.Errorf("error reading spec file %s: %v", specFile, err) + } + testSpecs := make(specs, 0) + err = json.Unmarshal(specBytes, &testSpecs) + if err != nil { + return nil, fmt.Errorf("error unmarshal specs: %v", err) + } + return testSpecs, nil +} + +func (tc *spec) executeSpecTest(t *testing.T) { + var ( + doc ast.Node + err error + ) + doc = gm.Parser().Parse(text.NewReader([]byte(tc.Markdown))) + if doc == nil { + t.Errorf("parsing example %d returns nil", tc.Example) + } + //r := markdown.NewLinkModifierRenderer() + buf := &bytes.Buffer{} + err = lmr.Render(buf, []byte(tc.Markdown), doc) + if err != nil { + fmt.Println("", tc.Example, tc.Markdown) + t.Errorf("render example %d fails with: %v", tc.Example, err) + } + _doc := gm.Parser().Parse(text.NewReader(buf.Bytes())) + if _doc == nil { + t.Errorf("parsing rendered example %d returns nil", tc.Example) + } + buf1, buf2 := &bytes.Buffer{}, &bytes.Buffer{} + err = gm.Convert([]byte(tc.Markdown), buf1) + if err != nil { + t.Errorf("convert example %d fails: %v", tc.Example, err) + } + err = gm.Convert(buf.Bytes(), buf2) + if err != nil { + t.Errorf("convert rendered example %d fails: %v", tc.Example, err) + } + if bytes.Compare(buf1.Bytes(), buf2.Bytes()) != 0 { + // try with tc HTML + if bytes.Compare([]byte(tc.HTML), buf2.Bytes()) != 0 { + // try to normalize HTMLs + var hn1, hn2 *html.Node + hn1, err = html.Parse(bytes.NewBuffer([]byte(tc.HTML))) + if err != nil { + t.Errorf("parse HTML for example %d fails: %v", tc.Example, err) + } + hn2, err = html.Parse(buf2) + if err != nil { + t.Errorf("parse HTML for rendered example %d fails: %v", tc.Example, err) + } + hBuf1, hBuf2 := &bytes.Buffer{}, &bytes.Buffer{} + err = html.Render(hBuf1, hn1) + if err != nil { + t.Errorf("render HTML node for example %d fails: %v", tc.Example, err) + } + err = html.Render(hBuf2, hn2) + if err != nil { + t.Errorf("render HTML node for rendered example %d fails: %v", tc.Example, err) + } + if bytes.Compare(hBuf1.Bytes(), hBuf2.Bytes()) != 0 { + t.Errorf("compare HTML results for example %d fails", tc.Example) + } + } + } +} diff --git a/pkg/processors/document.go b/pkg/processors/document.go deleted file mode 100644 index 02c5e4dc..00000000 --- a/pkg/processors/document.go +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package processors - -import ( - "github.com/gardener/docforge/pkg/api" -) - -// Document represents a markdown for the Node -type Document struct { - Node *api.Node - DocumentBytes []byte - Links []*Link - FrontMatter []byte -} - -// Link defines a markdown link -type Link struct { - DestinationNode *api.Node - IsResource bool - OriginalDestination string - AbsLink *string - Destination *string - Text *string - Title *string -} - -// Append adds content to the document -func (d *Document) Append(bytes []byte) { - if d.DocumentBytes == nil { - d.DocumentBytes = bytes - return - } - d.DocumentBytes = append(d.DocumentBytes, bytes...) -} - -// AddFrontMatter adds front matter content to the document -// TODO: adding to frontmatter multiple times should merge instead of override -func (d *Document) AddFrontMatter(frontMatter []byte) { - if d.FrontMatter == nil { - d.FrontMatter = make([]byte, 0) - } - d.FrontMatter = append(d.FrontMatter, frontMatter...) -} - -// AddLink adds link to the document -func (d *Document) AddLink(link *Link) { - if d.Links == nil { - d.Links = make([]*Link, 0) - } - - d.Links = append(d.Links, link) -} - -// GetLinkByDestination returns the links for a given destination -func (d *Document) GetLinkByDestination(destination string) *Link { - for _, l := range d.Links { - if *l.Destination == string(destination) { - return l - } - } - return nil -} diff --git a/pkg/processors/frontmatter.go b/pkg/processors/frontmatter.go deleted file mode 100644 index d2802883..00000000 --- a/pkg/processors/frontmatter.go +++ /dev/null @@ -1,115 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package processors - -import ( - "fmt" - "strings" - - "github.com/gardener/docforge/pkg/api" - "gopkg.in/yaml.v3" -) - -// FrontMatter is a processor implementation responsible to inject front-matter -// properties defined on nodes and default front-matter -type FrontMatter struct { - // IndexFileNames defines a list of file names that indicate - // their content can be used as section files. - IndexFileNames []string -} - -// Process implements Processor#Process -func (f *FrontMatter) Process(document *Document) error { - var ( - node = document.Node - - fmBytes []byte - props, fm, docFm map[string]interface{} - ok bool - err error - ) - - // Process only document nodes - if len(node.Source) == 0 && len(node.ContentSelectors) == 0 && node.Template == nil { - return nil - } - // Frontmatter from node - if props = node.Properties; props == nil { - props = map[string]interface{}{} - } - if _fm, found := props["frontmatter"]; found { - if fm, ok = _fm.(map[string]interface{}); !ok { - panic("node frontmatter type cast failed") - } - } - - if fm == nil { - // Minimal standard front matter, injected by default - // if no other has been defined on a node - // TODO: make this configurable option, incl. the default frontmatter we inject - fm = map[string]interface{}{} - } - - // This should be stripped earlier - // // document front matter - // if fmBytes, content, err = markdown.StripFrontMatter(documentBlob); err != nil { - // return nil, err - // } - fmBytes = document.FrontMatter - docFm = map[string]interface{}{} - if err := yaml.Unmarshal(fmBytes, docFm); err != nil { - return fmt.Errorf("failed to unmarsal the document frontmatter. %+v", err) - } - - for propertyKey, propertyValue := range docFm { - if _, ok := fm[propertyKey]; !ok { - fm[propertyKey] = propertyValue - } - } - - // TODO: merge node + doc frontmatter per configurable strategy: - // - merge where node frontmatter entries win over document frontmatter - // - merge where document frontmatter entries win over node frontmatter - // - merge where document frontmatter are merged with node frontmatter ignoring duplicates (currently impl.) - - if _, ok := fm["title"]; !ok { - fm["title"] = f.getNodeTitle(node) - } - - document.FrontMatter, err = yaml.Marshal(fm) - if err != nil { - return err - } - - return nil -} - -// Determines node title from its name or its parent name if -// it is eligible to be index file, and then normalizes either -// as a title - removing `-`, `_`, `.md` and converting to title -// case. -func (f *FrontMatter) getNodeTitle(node *api.Node) string { - title := node.Name - if node.Parent() != nil && f.nodeIsIndexFile(node.Name) { - title = node.Parent().Name - } - title = strings.TrimRight(title, ".md") - title = strings.ReplaceAll(title, "_", " ") - title = strings.ReplaceAll(title, "-", " ") - title = strings.Title(title) - return title -} - -// Compares a node name to the configured list of index file -// and a default name '_index.md' to determin if this node -// is an index document node. -func (f *FrontMatter) nodeIsIndexFile(name string) bool { - for _, s := range f.IndexFileNames { - if strings.EqualFold(name, s) { - return true - } - } - return name == "_index.md" -} diff --git a/pkg/processors/frontmatter_test.go b/pkg/processors/frontmatter_test.go deleted file mode 100644 index 89aca299..00000000 --- a/pkg/processors/frontmatter_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package processors - -import ( - "testing" - - "github.com/gardener/docforge/pkg/api" -) - -func TestFrontmatterProcess(t *testing.T) { - testCases := []struct { - name string - strippedFrontMatter string - node *api.Node - wantErr error - wantDocument string - mutate func(node *api.Node) - }{ - { - name: "adds_missing_title_to_document_front_matter", - strippedFrontMatter: "", - node: &api.Node{ - Source: "whatever", - Name: "test-1_2.md", - }, - wantErr: nil, - wantDocument: `title: Test 1 2`, - }, - { - name: "reuses_defined_title_in_document", - strippedFrontMatter: "# Heading", - node: &api.Node{Name: "test", Properties: map[string]interface{}{"frontmatter": map[string]interface{}{"title": "Test1"}}, Source: "whatever"}, - wantErr: nil, - wantDocument: `title: Test1`, - mutate: func(node *api.Node) { - }, - }, - { - name: "merge_front_matter_from_doc_and_node", - strippedFrontMatter: `prop1: A`, - node: &api.Node{Name: "test", Properties: map[string]interface{}{"frontmatter": map[string]interface{}{"title": "Test"}}, Source: "whatever"}, - wantErr: nil, - wantDocument: `prop1: A -title: Test`, - mutate: func(node *api.Node) { - }, - }, - { - name: "rewrite_title_defined_in_the_stripped_fm", - strippedFrontMatter: `title: Test1`, - node: &api.Node{ - Name: "test3", - Properties: map[string]interface{}{ - "frontmatter": map[string]interface{}{ - "title": "Test2", - }, - }, - Source: "whatever", - }, - wantErr: nil, - wantDocument: `title: Test2`, - }, - } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - fm := &FrontMatter{ - IndexFileNames: []string{"README.md"}, - } - if tc.mutate != nil { - tc.mutate(tc.node) - } - document := &Document{ - Node: tc.node, - FrontMatter: []byte(tc.strippedFrontMatter), - } - err := fm.Process(document) - if err != tc.wantErr { - t.Errorf("expected err %v!=%v", tc.wantErr, err) - } - if string(document.FrontMatter) == tc.wantDocument { - t.Errorf("expected bytes \n%s\n!=\n%s\n", tc.wantDocument, string(document.FrontMatter)) - } - }) - } -} diff --git a/pkg/processors/processor.go b/pkg/processors/processor.go deleted file mode 100644 index 2e3815b7..00000000 --- a/pkg/processors/processor.go +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package processors - -// Processor is used by extensions to transform a document -type Processor interface { - Process(document *Document) error -} - -// ProcessorChain is a registry of ordered document processors -// that implements Processor#Process -type ProcessorChain struct { - Processors []Processor -} - -// Process implements Processor#Process invoking the registered chain of Processors sequentially -func (p *ProcessorChain) Process(document *Document) error { - var err error - for _, p := range p.Processors { - if err := p.Process(document); err != nil { - return err - } - } - return err -} diff --git a/pkg/reactor/content_processor.go b/pkg/reactor/content_processor.go index fbafebec..ee1aed5d 100644 --- a/pkg/reactor/content_processor.go +++ b/pkg/reactor/content_processor.go @@ -5,271 +5,420 @@ package reactor import ( + "bytes" "context" "fmt" "github.com/gardener/docforge/pkg/api" "github.com/gardener/docforge/pkg/markdown" - "github.com/gardener/docforge/pkg/markdown/parser" - "github.com/gardener/docforge/pkg/processors" "github.com/gardener/docforge/pkg/resourcehandlers" "github.com/gardener/docforge/pkg/util/urls" - "github.com/hashicorp/go-multierror" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/renderer" "k8s.io/klog/v2" "net/url" "strings" "sync" + "text/template" ) -// NodeContentProcessor operates on documents content to reconcile links and -// schedule linked resources downloads +var ( + // pool with reusable buffers + bufPool = sync.Pool{ + New: func() interface{} { + return new(bytes.Buffer) + }, + } +) + +// NodeContentProcessor operates on documents content to reconcile links and schedule linked resources downloads type NodeContentProcessor interface { - ReconcileLinks(ctx context.Context, document *processors.Document, contentSource string, documentContents []byte) ([]byte, error) + // Process node content and write the result in a buffer + Process(ctx context.Context, buffer *bytes.Buffer, reader Reader, node *api.Node) error } type nodeContentProcessor struct { - resourceAbsLinks map[string]string - rwlock sync.RWMutex - globalLinksConfig *api.Links - // resourcesRoot specifies the root location for downloaded resource. - // It is used to rewrite resource links in documents to relative paths. resourcesRoot string - downloadScheduler DownloadScheduler + globalLinksConfig *api.Links + downloader DownloadScheduler validator Validator - failFast bool rewriteEmbedded bool resourceHandlers resourcehandlers.Registry sourceLocations map[string][]*api.Node + hugo bool + PrettyUrls bool + IndexFileNames []string + BaseURL string + mux sync.Mutex + resourceAbsLinks map[string]string + templates map[string]*template.Template + rwLock sync.RWMutex } // NewNodeContentProcessor creates NodeContentProcessor objects -func NewNodeContentProcessor(resourcesRoot string, globalLinksConfig *api.Links, downloadJob DownloadScheduler, validator Validator, failFast bool, rewriteEmbedded bool, resourceHandlers resourcehandlers.Registry) NodeContentProcessor { +func NewNodeContentProcessor(resourcesRoot string, globalLinksConfig *api.Links, downloadJob DownloadScheduler, validator Validator, rewriteEmbedded bool, rh resourcehandlers.Registry, + hugo bool, PrettyUrls bool, IndexFileNames []string, BaseURL string) NodeContentProcessor { c := &nodeContentProcessor{ - resourceAbsLinks: make(map[string]string), - globalLinksConfig: globalLinksConfig, + // resourcesRoot specifies the root location for downloaded resource. + // It is used to rewrite resource links in documents to relative paths. resourcesRoot: resourcesRoot, - downloadScheduler: downloadJob, + globalLinksConfig: globalLinksConfig, + downloader: downloadJob, validator: validator, - failFast: failFast, rewriteEmbedded: rewriteEmbedded, - resourceHandlers: resourceHandlers, + resourceHandlers: rh, + hugo: hugo, + PrettyUrls: PrettyUrls, + IndexFileNames: IndexFileNames, + BaseURL: BaseURL, sourceLocations: make(map[string][]*api.Node), + resourceAbsLinks: make(map[string]string), + templates: map[string]*template.Template{}, } return c } -//convenience wrapper -func (c *nodeContentProcessor) schedule(download *DownloadTask) error { - return c.downloadScheduler.Schedule(download) -} - -// ReconcileLinks analyzes a document referenced by a node's ContentSourcePath -// and processes its links to other resources to resolve their inconsistencies. -// The processing might involve rewriting links to relative and having new -// destinations, or rewriting them to absolute, as well as downloading some of -// the linked resources. -// The function returns the processed document or error. -func (c *nodeContentProcessor) ReconcileLinks(ctx context.Context, document *processors.Document, contentSourcePath string, documentContents []byte) ([]byte, error) { - klog.V(6).Infof("[%s] Reconciling links for %s\n", document.Node.Name, contentSourcePath) - parser := parser.NewParser() - parsedDocument := parser.Parse(documentContents) - //TODO: test out with older version what happens with multiple content selectors that have frontmatter - documentBytes, err := c.reconcileMDLinks(ctx, document, parsedDocument, contentSourcePath) - if err != nil { - return nil, err - } - if documentBytes, err = c.reconcileHTMLLinks(ctx, document, documentBytes, contentSourcePath); err != nil { - return nil, err - } - return documentBytes, err -} - -func (c *nodeContentProcessor) reconcileMDLinks(ctx context.Context, document *processors.Document, parsedDocument parser.Document, contentSourcePath string) ([]byte, error) { - var errors *multierror.Error - contentBytes, _ := markdown.UpdateMarkdownLinks(parsedDocument, func(markdownType markdown.Type, destination, text, title []byte) ([]byte, []byte, []byte, error) { - var ( - link *processors.Link - download *DownloadTask - err error - ) - // quick sanity check for ill-parsed links if any - if destination == nil { - klog.V(6).Infof("[%s] skipping ill parsed link: destination[%s] text[%s] title[%s]", contentSourcePath, string(destination), string(text), string(title)) - return destination, text, title, nil - } - if link, download, err = c.resolveLink(ctx, document, string(destination), contentSourcePath); err != nil { - errors = multierror.Append(errors, err) - if c.failFast { - return destination, text, title, err +func (c *nodeContentProcessor) Process(ctx context.Context, b *bytes.Buffer, r Reader, n *api.Node) error { + // api.Node content by priority + var nc []*docContent + path := api.Path(n, "/") + // 1. Process Source + if len(n.Source) > 0 { + source, err := r.Read(ctx, n.Source) + if err != nil { + if resourceNotFound, ok := err.(resourcehandlers.ErrResourceNotFound); ok { + klog.Warningf("reading source %s from node %s/%s failed: %s\n", n.Source, path, n.Name, resourceNotFound) + } else { + return fmt.Errorf("reading source %s from node %s/%s failed: %w", n.Source, path, n.Name, err) } } - if download != nil { - link.IsResource = true - if err := c.schedule(download); err != nil { - return destination, text, title, err + if len(source) == 0 { + klog.Warningf("no content read from node %s/%s source %s\n", path, n.Name, n.Source) + } else { + dc := &docContent{docCnt: source, docURI: n.Source} + dc.docAst, err = markdown.Parse(source) + if err != nil { + return fmt.Errorf("fail to parse source %s from node %s/%s: %w", n.Source, path, n.Name, err) } + nc = append(nc, dc) } - // rewrite abs links to embedded images to their raw format if necessary, to - // ensure they are embeddable - if c.rewriteEmbedded && markdownType == markdown.Image && link.Destination != nil { - if err = c.rawImage(link.Destination); err != nil { - return destination, text, title, err + } + // 2. Process ContentSelectors + if len(n.ContentSelectors) > 0 { + for _, content := range n.ContentSelectors { + source, err := r.Read(ctx, content.Source) + if err != nil { + if resourceNotFound, ok := err.(resourcehandlers.ErrResourceNotFound); ok { + klog.Warningf("reading content selector source %s from node %s/%s failed: %s\n", content.Source, path, n.Name, resourceNotFound) + } else { + return fmt.Errorf("reading content selector source %s from node %s/%s failed: %w", content.Source, path, n.Name, err) + } + } + if len(source) == 0 { + klog.Warningf("no content read from node %s/%s content selector source %s\n", path, n.Name, content.Source) + } else { + dc := &docContent{docCnt: source, docURI: content.Source} + dc.docAst, err = markdown.Parse(source) + if err != nil { + return fmt.Errorf("fail to parse content selector source %s from node %s/%s: %w", content.Source, path, n.Name, err) + } + nc = append(nc, dc) } } - // write node processing stats for document nodes - // TODO: separate stats logic below - if document.Node != nil { - if link.Destination != nil && *link.Destination != string(destination) { - if len(*link.Destination) == 0 { - *link.Destination = "*deleted*" + } + // 3. Process Template + var tmplBlob []byte + if n.Template != nil { + // init template + tmpl, err := c.initTemplate(ctx, path, r, n) + if err != nil { + return err + } + // init placeholders content + vars := map[string]string{} + for varName, content := range n.Template.Sources { + var source []byte + var doc ast.Node + source, err = r.Read(ctx, content.Source) + if err != nil { + if resourceNotFound, ok := err.(resourcehandlers.ErrResourceNotFound); ok { + klog.Warningf("reading template source %s:%s from node %s/%s failed: %s\n", varName, content.Source, path, n.Name, resourceNotFound) + } else { + return fmt.Errorf("reading template source %s:%s from node %s/%s failed: %w", varName, content.Source, path, n.Name, err) } - recordLinkStats(document.Node, "Links", fmt.Sprintf("%s -> %s", string(destination), *link.Destination)) + } + if len(source) == 0 { + klog.Warningf("no content read from node %s/%s template source %s:%s\n", path, n.Name, varName, content.Source) } else { - if link.Text != nil && len(*link.Text) == 0 { - recordLinkStats(document.Node, "Links", fmt.Sprintf("%s -> *deleted*", string(destination))) + doc, err = markdown.Parse(source) + if err != nil { + return fmt.Errorf("fail to parse template source %s:%s from node %s/%s: %w", varName, content.Source, path, n.Name, err) + } + } + if doc != nil { + // remove frontmatter for template placeholder values + if doc.Kind() == ast.KindDocument { + doc.(*ast.Document).SetMeta(nil) + } + // render content + var value string + value, err = c.renderTemplatePlaceholderValue(source, doc, n, content.Source) + if err != nil { + return fmt.Errorf("fail to render template source %s:%s from node %s/%s: %w", varName, content.Source, path, n.Name, err) } + vars[varName] = value + } else { + // add empty string + vars[varName] = "" } } - if link.Text != nil { - text = []byte(*link.Text) + if tmplBlob, err = applyTemplate(tmpl, vars, path, n); err != nil { + return err } - if link.Title != nil { - title = []byte(*link.Title) + // add empty doc content to prepend node frontmatter if only a template is defined + dc := &docContent{docAst: ast.NewDocument(), docCnt: []byte{}, docURI: ""} + nc = append(nc, dc) + } + // if no content -> return + if len(nc) == 0 { + return nil + } + // render node content + // 1 - frontmatter preprocessing + var fmp *frontmatterProcessor + if c.hugo { + fmp = &frontmatterProcessor{ + node: n, + IndexFileNames: c.IndexFileNames, } - if link.Destination == nil { - return nil, text, title, nil + } + if err := preprocessFrontmatter(nc, fmp); err != nil { + return err + } + // 2. - write node content + for _, cnt := range nc { + rnd := c.getRenderer(n, cnt.docURI) + if err := rnd.Render(b, cnt.docCnt, cnt.docAst); err != nil { + return err } - return []byte(*link.Destination), text, title, nil - }) - if c.failFast && errors != nil && errors.Len() > 0 { - return nil, errors.ErrorOrNil() } + if len(tmplBlob) > 0 { + _, _ = b.Write(tmplBlob) + } + return nil +} - return contentBytes, errors.ErrorOrNil() +type docContent struct { + docAst ast.Node + docCnt []byte + docURI string } -// replace html raw links of any sorts. -func (c *nodeContentProcessor) reconcileHTMLLinks(ctx context.Context, document *processors.Document, documentBytes []byte, contentSourcePath string) ([]byte, error) { - var ( - docNode = document.Node - errors *multierror.Error - destination *string - ) - documentBytes, _ = markdown.UpdateHTMLLinksRefs(documentBytes, func(isImage bool, url []byte) ([]byte, error) { - link, download, err := c.resolveLink(ctx, document, string(url), contentSourcePath) - destination = link.Destination - if err != nil { - errors = multierror.Append(errors, err) - return url, nil - } - if docNode != nil && destination != nil { - if string(url) != *destination { - recordLinkStats(docNode, "Links", fmt.Sprintf("%s -> %s", url, *destination)) - } else { - recordLinkStats(docNode, "Links", "") +func preprocessFrontmatter(nc []*docContent, fmp *frontmatterProcessor) error { + if len(nc) > 1 { + aggregated := make(map[string]interface{}) + for i := len(nc) - 1; i >= 0; i-- { + if nc[i].docAst.Kind() == ast.KindDocument { + d := nc[i].docAst.(*ast.Document) + mergeFrontmatter(aggregated, d.Meta()) + d.SetMeta(nil) } } - if download != nil { - link.IsResource = true - if err = c.schedule(download); err != nil { - errors = multierror.Append(errors, err) - return []byte(*destination), nil - } + if nc[0].docAst.Kind() == ast.KindDocument { + nc[0].docAst.(*ast.Document).SetMeta(aggregated) + } else { + // TODO: doc kind must be 'ast.KindDocument', where to validate this? } - // rewrite abs links to embedded images to their raw format if necessary, to - // ensure they are embeddable - if c.rewriteEmbedded && isImage && link.Destination != nil { - if err = c.rawImage(link.Destination); err != nil { - errors = multierror.Append(errors, err) - if c.failFast { - return []byte(*destination), err - } - } + } + if fmp != nil && len(nc) > 0 && nc[0].docAst.Kind() == ast.KindDocument { + d := nc[0].docAst.(*ast.Document) + pfm, err := fmp.processFrontmatter(d.Meta()) + if err != nil { + return err } - return []byte(*destination), nil - }) - return documentBytes, errors.ErrorOrNil() + d.SetMeta(pfm) + } + return nil } -// returns destination, text (alt-text for images), title, download(url, downloadName), err -func (c *nodeContentProcessor) resolveLink(ctx context.Context, document *processors.Document, destination string, contentSourcePath string) (*processors.Link, *DownloadTask, error) { - var ( - substituteDestination, version *string - downloadResourceName, absLink string - ok bool - globalRewrites map[string]*api.LinkRewriteRule - ) +// write add key:values over base +func mergeFrontmatter(base, add map[string]interface{}) { + for k, v := range add { + base[k] = v + } +} + +// TODO: how to ensure that template will produce valid Markdown ? +func applyTemplate(tmpl *template.Template, vars map[string]string, nodePath string, n *api.Node) ([]byte, error) { + tmplBytes := &bytes.Buffer{} + if err := tmpl.Execute(tmplBytes, vars); err != nil { + return nil, fmt.Errorf("executing template %s from node %s/%s failed: %w", n.Template.Path, nodePath, n.Name, err) + } + return tmplBytes.Bytes(), nil +} + +func (c *nodeContentProcessor) renderTemplatePlaceholderValue(source []byte, doc ast.Node, n *api.Node, sourceURI string) (string, error) { + rnd := c.getRenderer(n, sourceURI) + bytesBuff := bufPool.Get().(*bytes.Buffer) + defer bufPool.Put(bytesBuff) + bytesBuff.Reset() + if err := rnd.Render(bytesBuff, source, doc); err != nil { + return "", err + } + return string(bytesBuff.Bytes()), nil +} - node := document.Node - link := &processors.Link{ - OriginalDestination: destination, - Destination: &destination, +func (c *nodeContentProcessor) getTemplate(templatePath string) (*template.Template, bool) { + c.rwLock.RLock() + defer c.rwLock.RUnlock() + tmpl, ok := c.templates[templatePath] + return tmpl, ok +} + +func (c *nodeContentProcessor) initTemplate(ctx context.Context, nodePath string, r Reader, n *api.Node) (*template.Template, error) { + if tmpl, ok := c.getTemplate(n.Template.Path); ok { + if tmpl == nil { + return nil, fmt.Errorf("template %s from node %s/%s initialization failed", n.Template.Path, nodePath, n.Name) + } + return tmpl, nil + } + c.rwLock.Lock() + defer c.rwLock.Unlock() + if tmpl, ok := c.templates[n.Template.Path]; ok { + return tmpl, nil + } + blob, err := r.Read(ctx, n.Template.Path) + if err != nil { + if resourceNotFound, ok := err.(resourcehandlers.ErrResourceNotFound); ok { + klog.Warningf("reading template blob %s from node %s/%s failed: %s\n", n.Template.Path, nodePath, n.Name, resourceNotFound) + } else { + return nil, fmt.Errorf("reading template blob %s from node %s/%s failed: %w", n.Template.Path, nodePath, n.Name, err) + } + } + if len(blob) == 0 { + c.templates[n.Template.Path] = nil + return nil, fmt.Errorf("no content read from node %s/%s template blob %s", nodePath, n.Name, n.Template.Path) + } + var tmpl *template.Template + tmpl, err = template.New(n.Template.Path).Parse(string(blob)) + if err != nil { + c.templates[n.Template.Path] = nil + return nil, err } - document.AddLink(link) - if strings.HasPrefix(destination, "#") || strings.HasPrefix(destination, "mailto:") { - return link, nil, nil + c.templates[n.Template.Path] = tmpl + return tmpl, nil +} + +func (c *nodeContentProcessor) getRenderer(n *api.Node, sourceURI string) renderer.Renderer { + lr := c.newLinkResolver(n, sourceURI) + return markdown.NewLinkModifierRenderer(markdown.WithLinkResolver(lr.resolveLink)) +} + +// extends nodeContentProcessor with current source URI & node +type linkResolver struct { + *nodeContentProcessor + node *api.Node + source string +} + +func (c *nodeContentProcessor) newLinkResolver(node *api.Node, sourceURI string) *linkResolver { + return &linkResolver{ + nodeContentProcessor: c, + node: node, + source: sourceURI, } +} +// implements markdown.ResolveLink +func (l *linkResolver) resolveLink(dest string, isEmbeddable bool) (string, error) { + baseLink, err := l.resolveBaseLink(dest) + if err != nil { + return "", err + } + if isEmbeddable { + if l.rewriteEmbedded { + if err = l.rawImage(baseLink.Destination); err != nil { + return *baseLink.Destination, err + } + } + } + // TODO: include stats? + if l.hugo { + err = l.rewriteDestination(baseLink) + } + return *baseLink.Destination, err +} + +// resolve base link +func (l *linkResolver) resolveBaseLink(dest string) (*Link, error) { + link := &Link{ + OriginalDestination: dest, + Destination: &dest, + } + if strings.HasPrefix(dest, "#") || strings.HasPrefix(dest, "mailto:") { + return link, nil + } // validate destination - u, err := urls.Parse(destination) + u, err := urls.Parse(dest) if err != nil { - link.Destination = nil - return link, nil, err + return link, err } + // build absolute link + var absLink string if u.IsAbs() { // can we handle changes to this destination? - if c.resourceHandlers.Get(destination) == nil { + if l.resourceHandlers.Get(dest) == nil { // we don't have a handler for it. Leave it be. - c.validator.ValidateLink(u, destination, contentSourcePath) - return link, nil, err + l.validator.ValidateLink(u, dest, l.source) + return link, nil } - absLink = destination - } - if len(absLink) == 0 { - handler := c.resourceHandlers.Get(contentSourcePath) + absLink = dest + } else { + handler := l.resourceHandlers.Get(l.source) if handler == nil { - return link, nil, fmt.Errorf("no suitable handler registered for URL %s", contentSourcePath) + // TODO: here handler must exist because source was read + return link, fmt.Errorf("no suitable handler registered for URL %s", l.source) } // build absolute path for the destination using ContentSourcePath as base - if absLink, err = handler.BuildAbsLink(contentSourcePath, destination); err != nil { - if _, ok = err.(resourcehandlers.ErrResourceNotFound); ok { - klog.Warningf("failed to validate absolute link for %s from source %s: %v\n", destination, contentSourcePath, err) + if absLink, err = handler.BuildAbsLink(l.source, dest); err != nil { + if _, ok := err.(resourcehandlers.ErrResourceNotFound); ok { + klog.Warningf("failed to validate absolute link for %s from source %s: %v\n", dest, l.source, err) err = nil - if destination != absLink { + if dest != absLink { link.Destination = &absLink - klog.V(6).Infof("[%s] %s -> %s\n", contentSourcePath, destination, absLink) + klog.V(6).Infof("[%s] %s -> %s\n", l.source, dest, absLink) } } - return link, nil, err + return link, err } - link.AbsLink = &absLink } - // rewrite link if required - if gLinks := c.globalLinksConfig; gLinks != nil { + // rewrite link if required - TODO: remove link rewrites using rewrite rules? + var globalRewrites map[string]*api.LinkRewriteRule + if gLinks := l.globalLinksConfig; gLinks != nil { globalRewrites = gLinks.Rewrites } _a := absLink - if node != nil { - if version, substituteDestination, link.Text, link.Title, ok = MatchForLinkRewrite(absLink, node, globalRewrites); ok { + if l.node != nil { + if version, substituteDestination, _, _, ok := MatchForLinkRewrite(absLink, l.node, globalRewrites); ok { if substituteDestination != nil { if len(*substituteDestination) == 0 { // quit early. substitution is a request to remove this link s := "" - link.Text = &s - return link, nil, nil + link.Destination = &s + return link, nil } absLink = *substituteDestination } if version != nil { - handler := c.resourceHandlers.Get(absLink) + handler := l.resourceHandlers.Get(absLink) if handler == nil { link.Destination = &absLink - return link, nil, nil + return link, nil } - absLink, err = handler.SetVersion(absLink, *version) + absLink, err = handler.SetVersion(absLink, *version) // TODO: SetVersion? if err != nil { klog.Warningf("Failed to set version %s to %s: %s\n", *version, absLink, err.Error()) link.Destination = &absLink - return link, nil, nil + return link, nil } } } @@ -278,92 +427,160 @@ func (c *nodeContentProcessor) resolveLink(ctx context.Context, document *proces // validate potentially rewritten links u, err = urls.Parse(absLink) if err != nil { - return link, nil, err + return link, err } if _a != absLink { - klog.V(6).Infof("[%s] Link rewritten %s -> %s\n", contentSourcePath, _a, absLink) + klog.V(6).Infof("[%s] Link rewritten %s -> %s\n", l.source, _a, absLink) } - if node != nil { + if l.node != nil { // Links to other documents are enforced relative when // linking documents from the node structure. // Check if md extension to reduce the walkthroughs if u.Extension == "md" || u.Extension == "" { // try to find the link in source locations - if nl, ok := c.sourceLocations[strings.TrimSuffix(absLink, "/")]; ok { + if nl, ok := l.sourceLocations[strings.TrimSuffix(absLink, "/")]; ok { path := "" for _, n := range nl { - relPathBetweenNodes := node.RelativePath(n) + relPathBetweenNodes := l.node.RelativePath(n) if swapPaths(path, relPathBetweenNodes) { path = relPathBetweenNodes link.DestinationNode = n link.Destination = &relPathBetweenNodes } } - if destination != path { - klog.V(6).Infof("[%s] %s -> %s\n", contentSourcePath, destination, path) + if dest != path { + klog.V(6).Infof("[%s] %s -> %s\n", l.source, dest, path) } - return link, nil, nil + return link, nil } // a node with target source location does not exist -> rewrite destination with absolute link - if destination != absLink { + if dest != absLink { link.Destination = &absLink - klog.V(6).Infof("[%s] %s -> %s\n", contentSourcePath, destination, absLink) + klog.V(6).Infof("[%s] %s -> %s\n", l.source, dest, absLink) } - u, err = urls.Parse(*link.Destination) - c.validator.ValidateLink(u, destination, contentSourcePath) - return link, nil, nil + u, err = urls.Parse(absLink) + l.validator.ValidateLink(u, absLink, l.source) + return link, nil } - // Links to resources that are not structure document nodes are // assessed for download eligibility and if applicable their // destination is updated to relative path to predefined location // for resources. var globalDownloadsConfig *api.Downloads - if c.globalLinksConfig != nil { - globalDownloadsConfig = c.globalLinksConfig.Downloads + if l.globalLinksConfig != nil { + globalDownloadsConfig = l.globalLinksConfig.Downloads } - if downloadResourceName, ok = MatchForDownload(u, node, globalDownloadsConfig); ok { - resourceName := c.getDownloadResourceName(u, downloadResourceName) - _d := destination - destination = buildDownloadDestination(node, resourceName, c.resourcesRoot) - if _d != destination { - klog.V(6).Infof("[%s] %s -> %s\n", contentSourcePath, _d, destination) + if downloadResourceName, ok := MatchForDownload(u, l.node, globalDownloadsConfig); ok { + resourceName := l.getDownloadResourceName(u, downloadResourceName) + resLocation := buildDownloadDestination(l.node, resourceName, l.resourcesRoot) + if resLocation != dest { + link.Destination = &resLocation + klog.V(6).Infof("[%s] %s -> %s\n", l.source, resLocation, dest) } - link.Destination = &destination - return link, &DownloadTask{ + link.IsResource = true + if err = l.downloader.Schedule(&DownloadTask{ absLink, resourceName, - contentSourcePath, - _d, - }, nil + l.source, + dest, + }); err != nil { + return link, err + } + return link, nil } } - - if destination != absLink { + if dest != absLink { link.Destination = &absLink - klog.V(6).Infof("[%s] %s -> %s\n", contentSourcePath, destination, absLink) + klog.V(6).Infof("[%s] %s -> %s\n", l.source, dest, absLink) } - u, err = urls.Parse(*link.Destination) - c.validator.ValidateLink(u, destination, contentSourcePath) - return link, nil, nil + u, err = urls.Parse(absLink) + l.validator.ValidateLink(u, dest, l.source) + return link, nil } -func swapPaths(path string, newPath string) bool { - if path == "" { - return true +// used in Hugo mode +type frontmatterProcessor struct { + node *api.Node + IndexFileNames []string +} + +func (f *frontmatterProcessor) processFrontmatter(docFrontmatter map[string]interface{}) (map[string]interface{}, error) { + var nodeMeta, parentMeta map[string]interface{} + // 1 front matter from doc node + if val, ok := f.node.Properties["frontmatter"]; ok { + nodeMeta, ok = val.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid frontmatter properties for node: %s/%s", api.Path(f.node, "/"), f.node.Name) + } } - // prefer descending vs ascending paths - if !strings.HasPrefix(path, "./") && strings.HasPrefix(newPath, "./") { - return true + // 2 front matter from doc node parent (only if the current one is section file) + if f.node.Name == "_index.md" && f.node.Parent() != nil { + if val, ok := f.node.Parent().Properties["frontmatter"]; ok { + parentMeta, ok = val.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid frontmatter properties for node: %s", api.Path(f.node, "/")) + } + } } - // prefer shorter paths - return strings.Count(path, "/") > strings.Count(newPath, "/") + // overwrite docFrontmatter with node + for k, v := range nodeMeta { + docFrontmatter[k] = v + } + // overwrite docFrontmatter with parent + for k, v := range parentMeta { + docFrontmatter[k] = v + } + // add Title if missing + if _, ok := docFrontmatter["title"]; !ok { + docFrontmatter["title"] = f.getNodeTitle() + } + return docFrontmatter, nil +} + +// Determines node title from its name or its parent name if +// it is eligible to be index file, and then normalizes either +// as a title - removing `-`, `_`, `.md` and converting to title +// case. +func (f *frontmatterProcessor) getNodeTitle() string { + title := f.node.Name + if f.node.Parent() != nil && f.nodeIsIndexFile(f.node.Name) { + title = f.node.Parent().Name + } + title = strings.TrimRight(title, ".md") + title = strings.ReplaceAll(title, "_", " ") + title = strings.ReplaceAll(title, "-", " ") + title = strings.Title(title) + return title +} + +// Compares a node name to the configured list of index file +// and a default name '_index.md' to determin if this node +// is an index document node. +func (f *frontmatterProcessor) nodeIsIndexFile(name string) bool { + for _, s := range f.IndexFileNames { + if strings.EqualFold(name, s) { + return true + } + } + return name == "_index.md" +} + +// Check for cached resource name first and return that if found. Otherwise, +// return the downloadName +// TODO: +func (l *linkResolver) getDownloadResourceName(u *urls.URL, downloadName string) string { + l.mux.Lock() + defer l.mux.Unlock() + if cachedDownloadName, ok := l.resourceAbsLinks[u.Path]; ok { + return cachedDownloadName + } + return downloadName } // rewrite abs links to embedded objects to their raw link format if necessary, to // ensure they are embeddable -func (c *nodeContentProcessor) rawImage(link *string) (err error) { +func (l *linkResolver) rawImage(link *string) (err error) { var ( u *url.URL ) @@ -373,7 +590,7 @@ func (c *nodeContentProcessor) rawImage(link *string) (err error) { if !u.IsAbs() { return nil } - handler := c.resourceHandlers.Get(*link) + handler := l.resourceHandlers.Get(*link) if handler == nil { return nil } @@ -383,6 +600,94 @@ func (c *nodeContentProcessor) rawImage(link *string) (err error) { return nil } +func (l *linkResolver) rewriteDestination(lnk *Link) error { + link := *lnk.Destination + link = strings.TrimSpace(link) + if len(link) == 0 { + return nil + } + // trim leading and trailing quotes if any + link = strings.TrimSuffix(strings.TrimPrefix(link, "\""), "\"") + u, err := url.Parse(link) + if err != nil { + klog.Warning("Invalid link:", link) + return nil + } + if !u.IsAbs() && !strings.HasPrefix(link, "/") && !strings.HasPrefix(link, "#") { + _l := link + link = u.Path + if lnk.DestinationNode != nil { + absPath := api.Path(lnk.DestinationNode, "/") + link = fmt.Sprintf("%s/%s/%s", l.BaseURL, absPath, strings.ToLower(lnk.DestinationNode.Name)) + } + if lnk.IsResource { + for strings.HasPrefix(link, "../") { + link = strings.TrimPrefix(link, "../") + } + link = fmt.Sprintf("%s/%s", l.BaseURL, link) + } + + link = strings.TrimPrefix(link, "./") + if l.PrettyUrls { // TODO: This is HUGO cfg? what if the URLs are relative, do we need this modification? + link = strings.TrimSuffix(link, ".md") // TODO all index files names are set to _index.md + // Remove the last path segment if it is readme, index or _index + // The Hugo writer will rename those files to _index.md and runtime + // references will be to the sections in which they reside. + for _, s := range l.IndexFileNames { + if strings.HasSuffix(strings.ToLower(link), s) { + pathSegments := strings.Split(link, "/") + if len(pathSegments) > 0 { + pathSegments = pathSegments[:len(pathSegments)-1] + link = strings.Join(pathSegments, "/") + } + break + } + } + } else { + if strings.HasSuffix(link, ".md") { + link = strings.TrimSuffix(link, ".md") + // TODO: propagate fragment and query if any + link = fmt.Sprintf("%s.html", link) + } + } + if lnk.DestinationNode != nil { + // check for hugo url property & rewrite the link + // see https://gohugo.io/content-management/urls/ for details + if val, ok := lnk.DestinationNode.Properties["frontmatter"]; ok { // TODO: Remove such HUGO properties ??? + if fmProps, ok := val.(map[string]interface{}); ok { + if urlVal, ok := fmProps["url"]; ok { + if urlStr, ok := urlVal.(string); ok { + if _, err = url.Parse(urlStr); err != nil { + klog.Warningf("Invalid frontmatter url: %s for %s\n", urlStr, lnk.DestinationNode.Source) + } else { + link = urlStr + } + } + } + } + } + } + if _l != link { + klog.V(6).Infof("[%s] Rewriting node link for Hugo: %s -> %s \n", l.source, _l, link) + } + lnk.Destination = &link + return nil + } + return nil +} + +func swapPaths(path string, newPath string) bool { + if path == "" { + return true + } + // prefer descending vs ascending paths + if !strings.HasPrefix(path, "./") && strings.HasPrefix(newPath, "./") { + return true + } + // prefer shorter paths + return strings.Count(path, "/") > strings.Count(newPath, "/") +} + // Builds destination path for links from node to resource in root path // If root is not specified as document root (with leading "/"), the // returned destinations are relative paths from the node to the resource @@ -401,45 +706,11 @@ func buildDownloadDestination(node *api.Node, resourceName, root string) string return resourceRelPath } -// Check for cached resource name first and return that if found. Otherwise, -// return the downloadName -func (c *nodeContentProcessor) getDownloadResourceName(u *urls.URL, downloadName string) string { - c.rwlock.Lock() - defer c.rwlock.Unlock() - if cachedDownloadName, ok := c.resourceAbsLinks[u.Path]; ok { - return cachedDownloadName - } - return downloadName -} - -// recordLinkStats records link stats for a node -func recordLinkStats(node *api.Node, title, details string) { - var ( - stat *api.Stat - ) - nodeStats := node.GetStats() - for _, _stat := range nodeStats { - if _stat.Title == title { - stat = _stat - break - } - } - - if stat == nil { - stat = &api.Stat{ - Title: title, - } - if len(details) > 0 { - stat.Details = []string{details} - } else { - stat.Details = []string{} - } - stat.Figures = fmt.Sprintf("%d link rewrites", len(stat.Details)) - node.AddStats(stat) - return - } - if len(details) > 0 { - stat.Details = append(stat.Details, details) - } - stat.Figures = fmt.Sprintf("%d link rewrites", len(stat.Details)) +// Link defines a markdown link +type Link struct { + DestinationNode *api.Node + IsResource bool + OriginalDestination string + AbsLink *string + Destination *string } diff --git a/pkg/reactor/content_processor_test.go b/pkg/reactor/content_processor_test.go index a7337dae..16312d9c 100644 --- a/pkg/reactor/content_processor_test.go +++ b/pkg/reactor/content_processor_test.go @@ -5,20 +5,17 @@ package reactor import ( - "context" - "github.com/gardener/docforge/pkg/resourcehandlers/testhandler" - "github.com/gardener/docforge/pkg/util/urls" - "net/http" - "strings" - "testing" - "github.com/gardener/docforge/pkg/api" - "github.com/gardener/docforge/pkg/processors" "github.com/gardener/docforge/pkg/resourcehandlers" githubHandler "github.com/gardener/docforge/pkg/resourcehandlers/github" + "github.com/gardener/docforge/pkg/resourcehandlers/testhandler" "github.com/gardener/docforge/pkg/util/tests" + "github.com/gardener/docforge/pkg/util/urls" "github.com/google/go-github/v32/github" "github.com/stretchr/testify/assert" + "net/http" + "strings" + "testing" ) // TODO: This is a flaky test. In the future the ResourceHandler should be mocked. @@ -40,11 +37,11 @@ func Test_processLink(t *testing.T) { destination string contentSourcePath string wantDestination *string - wantText *string - wantTitle *string - wantDownload *DownloadTask - wantErr error - mutate func(c *nodeContentProcessor) + //wantText *string + //wantTitle *string + //wantDownload *DownloadTask + wantErr error + mutate func(c *nodeContentProcessor) }{ // skipped links { @@ -52,40 +49,40 @@ func Test_processLink(t *testing.T) { destination: "#internal-link", contentSourcePath: "", wantDestination: tests.StrPtr("#internal-link"), - wantText: nil, - wantTitle: nil, - wantDownload: nil, - wantErr: nil, + //wantText: nil, + //wantTitle: nil, + //wantDownload: nil, + wantErr: nil, }, { name: "mailto protocol is not processed", destination: "mailto:a@b.com", contentSourcePath: "", wantDestination: tests.StrPtr("mailto:a@b.com"), - wantText: nil, - wantTitle: nil, - wantDownload: nil, - wantErr: nil, + //wantText: nil, + //wantTitle: nil, + //wantDownload: nil, + wantErr: nil, }, { name: "Absolute links to releases not processed", destination: "https://github.com/gardener/gardener/releases/tag/v1.4.0", contentSourcePath: nodeA.Source, wantDestination: tests.StrPtr("https://github.com/gardener/gardener/releases/tag/v1.4.0"), - wantText: nil, - wantTitle: nil, - wantDownload: nil, - wantErr: nil, + //wantText: nil, + //wantTitle: nil, + //wantDownload: nil, + wantErr: nil, }, { name: "Relative links to releases not processed", destination: "../../../releases/tag/v1.4.0", contentSourcePath: nodeA.Source, wantDestination: tests.StrPtr("https://github.com/gardener/gardener/releases/tag/v1.4.0"), - wantText: nil, - wantTitle: nil, - wantDownload: nil, - wantErr: nil, + //wantText: nil, + //wantTitle: nil, + //wantDownload: nil, + wantErr: nil, }, // links to resources { @@ -93,10 +90,10 @@ func Test_processLink(t *testing.T) { destination: "./image.png", contentSourcePath: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", wantDestination: tests.StrPtr("https://github.com/gardener/gardener/blob/v1.10.0/docs/image.png"), - wantText: nil, - wantTitle: nil, - wantDownload: nil, - wantErr: nil, + //wantText: nil, + //wantTitle: nil, + //wantDownload: nil, + wantErr: nil, }, { name: "Relative link to resource in download scope", @@ -106,14 +103,14 @@ func Test_processLink(t *testing.T) { destination: "./image.png", contentSourcePath: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", wantDestination: tests.StrPtr("/__resources"), - wantText: nil, - wantTitle: nil, - wantDownload: &DownloadTask{ - Source: "https://github.com/gardener/gardener/blob/v1.10.0/docs/image.png", - Target: "", - Referer: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", - Reference: "./image.png", - }, + //wantText: nil, + //wantTitle: nil, + //wantDownload: &DownloadTask{ + // Source: "https://github.com/gardener/gardener/blob/v1.10.0/docs/image.png", + // Target: "", + // Referer: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", + // Reference: "./image.png", + //}, wantErr: nil, mutate: func(c *nodeContentProcessor) { h := testhandler.NewTestResourceHandler().WithAccept(func(uri string) bool { @@ -146,10 +143,10 @@ func Test_processLink(t *testing.T) { destination: "../image.png", contentSourcePath: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", wantDestination: tests.StrPtr("https://github.com/gardener/gardener/blob/v1.10.0/image.png"), - wantText: nil, - wantTitle: nil, - wantDownload: nil, - wantErr: nil, + //wantText: nil, + //wantTitle: nil, + //wantDownload: nil, + wantErr: nil, }, { name: "Absolute link to resource NOT in download scope", @@ -157,10 +154,10 @@ func Test_processLink(t *testing.T) { destination: "https://github.com/owner/repo/blob/master/docs/image.png", contentSourcePath: nodeA.Source, wantDestination: tests.StrPtr("https://github.com/owner/repo/blob/master/docs/image.png"), - wantText: nil, - wantTitle: nil, - wantDownload: nil, - wantErr: nil, + //wantText: nil, + //wantTitle: nil, + //wantDownload: nil, + wantErr: nil, }, { name: "Absolute link to resource in download scope", @@ -170,14 +167,14 @@ func Test_processLink(t *testing.T) { destination: "https://github.com/gardener/gardener/blob/v1.10.0/docs/image.png", contentSourcePath: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", wantDestination: tests.StrPtr("/__resources"), - wantText: nil, - wantTitle: nil, - wantDownload: &DownloadTask{ - Source: "https://github.com/gardener/gardener/blob/v1.10.0/docs/image.png", - Target: "", - Referer: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", - Reference: "https://github.com/gardener/gardener/blob/v1.10.0/docs/image.png", - }, + //wantText: nil, + //wantTitle: nil, + //wantDownload: &DownloadTask{ + // Source: "https://github.com/gardener/gardener/blob/v1.10.0/docs/image.png", + // Target: "", + // Referer: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", + // Reference: "https://github.com/gardener/gardener/blob/v1.10.0/docs/image.png", + //}, wantErr: nil, mutate: func(c *nodeContentProcessor) { c.globalLinksConfig = &api.Links{ @@ -201,10 +198,10 @@ func Test_processLink(t *testing.T) { destination: "https://github.com/owner/repo/blob/master/docs/doc.md", contentSourcePath: nodeA.Source, wantDestination: tests.StrPtr("https://github.com/owner/repo/blob/master/docs/doc.md"), - wantText: nil, - wantTitle: nil, - wantDownload: nil, - wantErr: nil, + //wantText: nil, + //wantTitle: nil, + //wantDownload: nil, + wantErr: nil, }, { name: "Absolute link to document in download scope and from structure", @@ -212,10 +209,10 @@ func Test_processLink(t *testing.T) { destination: nodeB.Source, contentSourcePath: nodeA.Source, wantDestination: tests.StrPtr("./node_B.md"), - wantText: nil, - wantTitle: nil, - wantDownload: nil, - wantErr: nil, + //wantText: nil, + //wantTitle: nil, + //wantDownload: nil, + wantErr: nil, mutate: func(c *nodeContentProcessor) { c.sourceLocations = map[string][]*api.Node{nodeB.Source: {nodeB}} c.globalLinksConfig = &api.Links{ @@ -238,10 +235,10 @@ func Test_processLink(t *testing.T) { destination: "https://github.com/owner/repo/blob/master/docs/doc.md", contentSourcePath: nodeA.Source, wantDestination: tests.StrPtr("https://github.com/owner/repo/blob/master/docs/doc.md"), - wantText: nil, - wantTitle: nil, - wantDownload: nil, - wantErr: nil, + //wantText: nil, + //wantTitle: nil, + //wantDownload: nil, + wantErr: nil, }, { name: "Relative link to document in download scope and NOT from structure", @@ -249,10 +246,10 @@ func Test_processLink(t *testing.T) { destination: "./another.md", contentSourcePath: nodeA.Source, wantDestination: tests.StrPtr("https://github.com/gardener/gardener/blob/v1.10.0/docs/another.md"), - wantText: nil, - wantTitle: nil, - wantDownload: nil, - wantErr: nil, + //wantText: nil, + //wantTitle: nil, + //wantDownload: nil, + wantErr: nil, mutate: func(c *nodeContentProcessor) { c.globalLinksConfig = &api.Links{ Rewrites: map[string]*api.LinkRewriteRule{ @@ -275,10 +272,10 @@ func Test_processLink(t *testing.T) { destination: "https://github.com/gardener/gardener/blob/master/sample.md", contentSourcePath: nodeA.Source, wantDestination: tests.StrPtr("https://github.com/gardener/gardener/blob/v1.10.0/sample.md"), - wantText: nil, - wantTitle: nil, - wantDownload: nil, - wantErr: nil, + //wantText: nil, + //wantTitle: nil, + //wantDownload: nil, + wantErr: nil, mutate: func(c *nodeContentProcessor) { c.globalLinksConfig = &api.Links{ Rewrites: map[string]*api.LinkRewriteRule{ @@ -295,14 +292,14 @@ func Test_processLink(t *testing.T) { destination: "https://github.com/gardener/gardener/blob/master/docs/image.png", contentSourcePath: nodeA.Source, wantDestination: tests.StrPtr("/__resources"), - wantText: nil, - wantTitle: nil, - wantDownload: &DownloadTask{ - Source: "https://github.com/gardener/gardener/blob/v1.10.0/docs/image.png", - Target: "", - Referer: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", - Reference: "https://github.com/gardener/gardener/blob/master/docs/image.png", - }, + //wantText: nil, + //wantTitle: nil, + //wantDownload: &DownloadTask{ + // Source: "https://github.com/gardener/gardener/blob/v1.10.0/docs/image.png", + // Target: "", + // Referer: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", + // Reference: "https://github.com/gardener/gardener/blob/master/docs/image.png", + //}, wantErr: nil, mutate: func(c *nodeContentProcessor) { c.globalLinksConfig = &api.Links{ @@ -335,14 +332,14 @@ func Test_processLink(t *testing.T) { destination: "./image.png", contentSourcePath: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", wantDestination: tests.StrPtr("/__resources"), - wantText: tests.StrPtr("Test text"), - wantTitle: tests.StrPtr("Test title"), - wantDownload: &DownloadTask{ - Source: "https://github.com/gardener/gardener/blob/v1.10.0/docs/image.png", - Target: "", - Referer: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", - Reference: "./image.png", - }, + //wantText: tests.StrPtr("Test text"), + //wantTitle: tests.StrPtr("Test title"), + //wantDownload: &DownloadTask{ + // Source: "https://github.com/gardener/gardener/blob/v1.10.0/docs/image.png", + // Target: "", + // Referer: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", + // Reference: "./image.png", + //}, wantErr: nil, mutate: func(c *nodeContentProcessor) { h := testhandler.NewTestResourceHandler().WithAccept(func(uri string) bool { @@ -376,26 +373,29 @@ func Test_processLink(t *testing.T) { resourceHandlers: resourcehandlers.NewRegistry(githubHandler.NewResourceHandler(github.NewClient(nil), http.DefaultClient, []string{"github.com"})), rewriteEmbedded: true, validator: &fakeValidator{}, + downloader: &fakeDownload{}, } if tc.mutate != nil { tc.mutate(c) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + //ctx, cancel := context.WithCancel(context.Background()) + //defer cancel() - document := &processors.Document{ - Node: tc.node, + lr := &linkResolver{ + nodeContentProcessor: c, + node: tc.node, + source: tc.contentSourcePath, } - link, gotDownload, gotErr := c.resolveLink(ctx, document, tc.destination, tc.contentSourcePath) + link, gotErr := lr.resolveBaseLink(tc.destination) assert.Equal(t, tc.wantErr, gotErr) - if gotDownload != nil { - if tc.wantDownload != nil { - tc.wantDownload.Target = gotDownload.Target - } - } - assert.Equal(t, tc.wantDownload, gotDownload) - var destination, text, title string + //if gotDownload != nil { + // if tc.wantDownload != nil { + // tc.wantDownload.Target = gotDownload.Target + // } + //} + //assert.Equal(t, tc.wantDownload, gotDownload) + var destination /*, text, title*/ string if link.Destination != nil { destination = *link.Destination } @@ -407,130 +407,22 @@ func Test_processLink(t *testing.T) { } else { assert.Equal(t, tc.wantDestination, link.Destination) } - if link.Text != nil { - text = *link.Text - } - if tc.wantText != nil { - assert.Equal(t, *tc.wantText, text) - } else { - assert.Equal(t, tc.wantText, link.Text) - } - if link.Title != nil { - title = *link.Title - } - if tc.wantText != nil { - assert.Equal(t, *tc.wantTitle, title) - } else { - assert.Equal(t, tc.wantTitle, link.Title) - } - }) - } -} - -func Test_matchHTMLLinks(t *testing.T) { - testCases := []struct { - name string - in string - want string - }{ - { - name: "rewrite_to_cde", - in: `<script src="abc/a.js" />`, - want: `<script src="cde" />`, - }, - { - name: "rewrite_to_cde_leave_whitespace_in_tag", - in: `<script src= "abc/a.js" />`, - want: `<script src= "cde" />`, - }, - { - name: "rewrite_to_cde_trim_whitespace_in_link", - in: `<script src=" abc/a.js" />`, - want: `<script src="cde" />`, - }, - { - name: "rewrite_to_cde_trim_whitespace_in_link", - in: `<script SRC = " abc/a.js" />`, - want: `<script SRC = "cde" />`, - }, - { - name: "rewrite_to_cde_trim_white_space_in_link_leave_whitespace_in_html_tag_single_quatation", - in: `<script SRC = ' abc/a.js' />`, - want: `<script SRC = 'cde' />`, - }, - { - name: "successful_rewrite_with_html_attributes", - in: `<script SRC = ' abc/a.js ' title="test" />`, - want: `<script SRC = 'cde' title="test" />`, - }, - { - name: "successful_rewrite_without_quatation", - in: `<script src= abc/a.js />`, - want: `<script src= cde />`, - }, - { - name: "rewrite_script_src", - in: `<script SRC = " abc/a.js" `, - want: `<script SRC = "cde" `, - }, - { - name: "rewrite_img_tags", - in: `<img src="abc/a.js">`, - want: `<img src="cde">`, - }, - { - name: "rewrite_href_with_surrounding_tag", - in: `<a href="abc/a.js">A</a>`, - want: `<a href="cde">A</a>`, - }, - { - name: "unmodified_if_no_tag", - in: `< src="abc/a.js" />`, - want: `< src="abc/a.js" />`, - }, - { - name: "unmodified_different_beggining_and_end_quatation", - in: `<script src=" abc/a.js' />`, - want: `<script src=" abc/a.js' />`, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - c := &nodeContentProcessor{ - resourceAbsLinks: make(map[string]string), - resourcesRoot: "/__resources", - resourceHandlers: resourcehandlers.NewRegistry(testhandler.NewTestResourceHandler().WithBuildAbsLink(func(source, link string) (string, error) { - return link, nil - }).WithGetRawFormatLink(func(absLink string) (string, error) { - return absLink, nil - }).WithSetVersion(func(absLink, version string) (string, error) { - return absLink, nil - })), - validator: &fakeValidator{}, - rewriteEmbedded: true, - globalLinksConfig: &api.Links{ - Rewrites: map[string]*api.LinkRewriteRule{ - "abc": { - Destination: tests.StrPtr("cde"), - }, - }, - }, - } - node := &api.Node{ - Name: "node_A.md", - Source: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md", - } - var ( - b []byte - err error - ) - doc := &processors.Document{ - Node: node, - } - if b, err = c.reconcileHTMLLinks(context.TODO(), doc, []byte(tc.in), ""); err != nil { - t.Fatal(err) - } - assert.Equal(t, tc.want, string(b)) + //if link.Text != nil { + // text = *link.Text + //} + //if tc.wantText != nil { + // assert.Equal(t, *tc.wantText, text) + //} else { + // assert.Equal(t, tc.wantText, link.Text) + //} + //if link.Title != nil { + // title = *link.Title + //} + //if tc.wantText != nil { + // assert.Equal(t, *tc.wantTitle, title) + //} else { + // assert.Equal(t, tc.wantTitle, link.Title) + //} }) } } @@ -540,3 +432,9 @@ type fakeValidator struct{} func (f *fakeValidator) ValidateLink(linkUrl *urls.URL, linkDestination, contentSourcePath string) bool { return true } + +type fakeDownload struct{} + +func (f fakeDownload) Schedule(task *DownloadTask) error { + return nil +} diff --git a/pkg/reactor/document_worker.go b/pkg/reactor/document_worker.go index 46b8dff8..30f2aec6 100644 --- a/pkg/reactor/document_worker.go +++ b/pkg/reactor/document_worker.go @@ -8,27 +8,17 @@ import ( "bytes" "context" "fmt" - "io/ioutil" - "sync" - "text/template" - "github.com/gardener/docforge/pkg/api" - "github.com/gardener/docforge/pkg/markdown" - "github.com/gardener/docforge/pkg/processors" - "github.com/gardener/docforge/pkg/resourcehandlers" "github.com/gardener/docforge/pkg/writers" "k8s.io/klog/v2" ) // DocumentWorker defines a structure for processing api.Node document content type DocumentWorker struct { - reader Reader - writer writers.Writer - processors.Processor + reader Reader + writer writers.Writer NodeContentProcessor NodeContentProcessor gitHubInfo GitHubInfo - templates map[string]*template.Template - rwLock sync.RWMutex } // DocumentWorkTask implements jobs#Task @@ -38,124 +28,28 @@ type DocumentWorkTask struct { // Work implements jobs.WorkerFunc func (w *DocumentWorker) Work(ctx context.Context, task interface{}) error { - var ( - documentBytes []byte - ) if dwTask, ok := task.(*DocumentWorkTask); ok { - if len(dwTask.Node.Nodes) == 0 { - // Node is considered a `Document Node` - var bytesBuff bytes.Buffer - doc := &processors.Document{ - Node: dwTask.Node, - } - // TODO: separate the logic of Document Processing out of the DocumentWorker - if len(dwTask.Node.ContentSelectors) > 0 { - for _, content := range dwTask.Node.ContentSelectors { - sourceBlob, err := w.reader.Read(ctx, content.Source) - if err != nil { - return err - } - if len(sourceBlob) == 0 { - continue - } - fm, sourceBlob, err := markdown.StripFrontMatter(sourceBlob) - if err != nil { - return err - } - doc.AddFrontMatter(fm) - if sourceBlob, err = w.NodeContentProcessor.ReconcileLinks(ctx, doc, content.Source, sourceBlob); err != nil { - return err - } - bytesBuff.Write(sourceBlob) - } - } - if dwTask.Node.Template != nil { - vars := map[string]string{} - for varName, content := range dwTask.Node.Template.Sources { - sourceBlob, err := w.reader.Read(ctx, content.Source) - if err != nil { - return err - } - if len(sourceBlob) == 0 { - continue - } - fm, sourceBlob, err := markdown.StripFrontMatter(sourceBlob) - if err != nil { - return err - } - doc.AddFrontMatter(fm) - if sourceBlob, err = w.NodeContentProcessor.ReconcileLinks(ctx, doc, content.Source, sourceBlob); err != nil { - return err - } - vars[varName] = string(sourceBlob) - } - var ( - templateBlob []byte - tmpl *template.Template - err error - ) - if tmpl = w.getTemplate(dwTask.Node.Template.Path); tmpl == nil { - if templateBlob, err = w.reader.Read(ctx, dwTask.Node.Template.Path); err != nil { - return err - } - if tmpl, err = template.New(dwTask.Node.Template.Path).Parse(string(templateBlob)); err != nil { - return err - } - w.setTemplate(dwTask.Node.Template.Path, tmpl) - } - if err := tmpl.Execute(&bytesBuff, vars); err != nil { - return err - } - } - if len(dwTask.Node.Source) > 0 { - sourceBlob, err := w.reader.Read(ctx, dwTask.Node.Source) - if err != nil { - if resourceNotFound, ok := err.(resourcehandlers.ErrResourceNotFound); ok { - klog.Warningf("reading %s failed: %s\n", dwTask.Node.Source, resourceNotFound) - return nil - } - return err - } - if len(sourceBlob) == 0 { - klog.Warningf("No content read from node %s source %s:", dwTask.Node.Name, dwTask.Node.Source) - return nil - } - fm, sourceBlob, err := markdown.StripFrontMatter(sourceBlob) - if err != nil { - return err - } - doc.AddFrontMatter(fm) - if sourceBlob, err = w.NodeContentProcessor.ReconcileLinks(ctx, doc, dwTask.Node.Source, sourceBlob); err != nil { - return err - } - bytesBuff.Write(sourceBlob) - } - if bytesBuff.Len() == 0 && len(doc.FrontMatter) == 0 { - klog.Warningf("Document node processing halted: No content assigned to document node %s", dwTask.Node.Name) - return nil - } - var err error - documentBytes, err = ioutil.ReadAll(&bytesBuff) - if err != nil { + var cnt []byte + path := api.Path(dwTask.Node, "/") + if len(dwTask.Node.Nodes) == 0 { // Node is considered a `Document Node` + // Process the node + bytesBuff := bufPool.Get().(*bytes.Buffer) + defer bufPool.Put(bytesBuff) + bytesBuff.Reset() + if err := w.NodeContentProcessor.Process(ctx, bytesBuff, w.reader, dwTask.Node); err != nil { return err } - doc.Append(documentBytes) - if w.Processor != nil { - if err := w.Processor.Process(doc); err != nil { - return fmt.Errorf("failed to process the document: %s. %+v", doc.Node.Source, err) - } - } - documentBytes = doc.DocumentBytes - if documentBytes, err = markdown.InsertFrontMatter(doc.FrontMatter, documentBytes); err != nil { - return fmt.Errorf("failed to insert frontmatter to document: %s. %+v", doc.Node.Source, err) + if bytesBuff.Len() == 0 { + klog.Warningf("document node processing halted: no content assigned to document node %s/%s", path, dwTask.Node.Name) + return nil } + cnt = bytesBuff.Bytes() } - path := api.Path(dwTask.Node, "/") - if err := w.writer.Write(dwTask.Node.Name, path, documentBytes, dwTask.Node); err != nil { + if err := w.writer.Write(dwTask.Node.Name, path, cnt, dwTask.Node); err != nil { return err } - if w.gitHubInfo != nil && len(documentBytes) > 0 { + if w.gitHubInfo != nil && len(cnt) > 0 { w.gitHubInfo.WriteGitHubInfo(dwTask.Node) } } else { @@ -163,18 +57,3 @@ func (w *DocumentWorker) Work(ctx context.Context, task interface{}) error { } return nil } - -func (w *DocumentWorker) getTemplate(name string) *template.Template { - w.rwLock.RLock() - defer w.rwLock.RUnlock() - if tmpl, ok := w.templates[name]; ok { - return tmpl - } - return nil -} - -func (w *DocumentWorker) setTemplate(name string, tmpl *template.Template) { - w.rwLock.Lock() - defer w.rwLock.Unlock() - w.templates[name] = tmpl -} diff --git a/pkg/reactor/integration_test.go b/pkg/reactor/integration_test.go deleted file mode 100644 index 016d8ff1..00000000 --- a/pkg/reactor/integration_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package reactor - -import ( - "context" - "flag" - "path/filepath" - "testing" - "time" - - "github.com/gardener/docforge/pkg/api" - "github.com/gardener/docforge/pkg/hugo" - "github.com/gardener/docforge/pkg/processors" - "github.com/gardener/docforge/pkg/resourcehandlers" - "github.com/gardener/docforge/pkg/resourcehandlers/github" - "github.com/gardener/docforge/pkg/util/tests" - "github.com/gardener/docforge/pkg/writers" - - githubapi "github.com/google/go-github/v32/github" - "golang.org/x/oauth2" -) - -// Run with: -// go test -timeout 30s -run ^_TestReactorWithGitHub$ -v -tags=integration --token=<your-token-here> - -var ghToken = flag.String("token", "", "GitHub personal token for authenticating requests") - -func init() { - tests.SetKlogV(6) -} - -func _TestReactorWithGitHub(t *testing.T) { - timeout := 300 * time.Second - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - docs := &api.Documentation{ - Structure: []*api.Node{ - { - Name: "docs", - NodeSelector: &api.NodeSelector{ - Path: "https://github.com/gardener/gardener/tree/v1.10.0/docs", - }, - Nodes: []*api.Node{ - { - Name: "calico", - NodeSelector: &api.NodeSelector{ - Path: "https://github.com/gardener/gardener-extension-networking-calico/tree/master/docs", - }, - }, - { - Name: "aws", - NodeSelector: &api.NodeSelector{ - Path: "https://github.com/gardener/gardener-extension-provider-aws/tree/master/docs", - }, - }, - }, - }, - }, - Links: &api.Links{ - Rewrites: map[string]*api.LinkRewriteRule{ - "gardener/gardener/(blob|tree|raw)": { - Version: tests.StrPtr("v1.10.0"), - }, - "gardener/gardener-extension-provider-aws/(blob|tree|raw)": { - Version: tests.StrPtr("v1.15.3"), - }, - "gardener/gardener-extension-networking-calico/(blob|tree|raw)": { - Version: tests.StrPtr("v1.10.0"), - }, - }, - Downloads: &api.Downloads{ - Scope: map[string]api.ResourceRenameRules{ - "gardener/gardener/(blob|tree|raw)/v1.10.0/docs": nil, - "gardener/gardener-extension-provider-aws/(blob|tree|raw)/v1.15.3/docs": nil, - "gardener/gardener-extension-networking-calico/(blob|tree|raw)/v1.10.0/docs": nil, - }, - }, - }, - } - - destination := "../../dev" - resourcesRoot := "/__resources" - ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: *ghToken}) - httpClient := oauth2.NewClient(ctx, ts) - ghClient := githubapi.NewClient(httpClient) - gh := github.NewResourceHandler(ghClient, httpClient, []string{"github.com"}) - writer := &writers.FSWriter{ - Root: destination, - } - hugoOptions := &hugo.Options{ - IndexFileNames: []string{"_index", "index", "readme", "read.me"}, - PrettyUrls: true, - Writer: writer, - } - options := &Options{ - MaxWorkersCount: 10, - MinWorkersCount: 5, - FailFast: true, - DestinationPath: destination, - ResourcesPath: resourcesRoot, - ResourceDownloadWorkersCount: 4, - Processor: &processors.ProcessorChain{ - Processors: []processors.Processor{ - &processors.FrontMatter{}, - hugo.NewProcessor(hugoOptions), - }, - }, - ResourceDownloadWriter: &writers.FSWriter{ - Root: filepath.Join(destination, resourcesRoot), - }, - Writer: hugo.NewWriter(hugoOptions), - ResourceHandlers: []resourcehandlers.ResourceHandler{gh}, - } - r, err := NewReactor(options) - if err != nil { - t.Errorf("failed with: %v", err) - } - - if err = r.Run(ctx, docs, false); err != nil { - t.Errorf("failed with: %v", err) - } - -} diff --git a/pkg/reactor/reactor.go b/pkg/reactor/reactor.go index 977737b5..0743246d 100644 --- a/pkg/reactor/reactor.go +++ b/pkg/reactor/reactor.go @@ -10,21 +10,16 @@ import ( "context" "errors" "fmt" + "github.com/gardener/docforge/pkg/api" + "github.com/gardener/docforge/pkg/jobs" + "github.com/gardener/docforge/pkg/resourcehandlers" + "github.com/gardener/docforge/pkg/writers" "io" + "k8s.io/klog/v2" "net/http" "os" "strings" "sync" - "text/template" - - "github.com/gardener/docforge/pkg/jobs" - - "github.com/gardener/docforge/pkg/processors" - "k8s.io/klog/v2" - - "github.com/gardener/docforge/pkg/api" - "github.com/gardener/docforge/pkg/resourcehandlers" - "github.com/gardener/docforge/pkg/writers" ) // Options encapsulates the parameters for creating @@ -38,15 +33,17 @@ type Options struct { ManifestAbsPath string ResourceDownloadWorkersCount int RewriteEmbedded bool - processors.Processor - ResourceDownloadWriter writers.Writer - GitInfoWriter writers.Writer - Writer writers.Writer - ResourceHandlers []resourcehandlers.ResourceHandler - DryRunWriter writers.DryRunWriter - Resolve bool - GlobalLinksConfig *api.Links - IndexFileNames []string + ResourceDownloadWriter writers.Writer + GitInfoWriter writers.Writer + Writer writers.Writer + ResourceHandlers []resourcehandlers.ResourceHandler + DryRunWriter writers.DryRunWriter + Resolve bool + GlobalLinksConfig *api.Links + Hugo bool + PrettyUrls bool + IndexFileNames []string + BaseURL string } // NewReactor creates a Reactor from Options @@ -89,10 +86,8 @@ func NewReactor(o *Options) (*Reactor, error) { worker := &DocumentWorker{ writer: o.Writer, reader: &GenericReader{ResourceHandlers: rhRegistry}, - NodeContentProcessor: NewNodeContentProcessor(o.ResourcesPath, o.GlobalLinksConfig, dScheduler, v, o.FailFast, o.RewriteEmbedded, rhRegistry), - Processor: o.Processor, + NodeContentProcessor: NewNodeContentProcessor(o.ResourcesPath, o.GlobalLinksConfig, dScheduler, v, o.RewriteEmbedded, rhRegistry, o.Hugo, o.PrettyUrls, o.IndexFileNames, o.BaseURL), gitHubInfo: ghInfo, - templates: map[string]*template.Template{}, } docTasks, err := jobs.NewJobQueue("Document", o.MaxWorkersCount, worker.Work, o.FailFast, reactorWG) if err != nil { @@ -175,6 +170,7 @@ func getSourceLocationsMap(structure []*api.Node) map[string][]*api.Node { return locations } +// TODO: add content selector & template locations ??? func addSourceLocation(locations map[string][]*api.Node, node *api.Node) { if node.Source != "" { locations[node.Source] = append(locations[node.Source], node) diff --git a/pkg/resourcehandlers/git/gitGinko_resource_handler_test.go b/pkg/resourcehandlers/git/gitGinko_resource_handler_test.go index 01ee593f..21a1d99a 100644 --- a/pkg/resourcehandlers/git/gitGinko_resource_handler_test.go +++ b/pkg/resourcehandlers/git/gitGinko_resource_handler_test.go @@ -159,12 +159,12 @@ var _ = Describe("Git", func() { tags = []string{"v4.9", "v5.7", "v6.1", "v7.7"} uri = "https://github.com/testOrg/testRepo/blob/DEFAULT_BRANCH/testManifest.yaml" var ( - fakeRepository gitinterfacefakes.FakeRepository - fakeWorktree *gitinterfacefakes.FakeRepositoryWorktree + fakeRepository = &gitinterfacefakes.FakeRepository{} + fakeWorktree = &gitinterfacefakes.FakeRepositoryWorktree{} ) //fakeGit.PlainCloneContextReturns(&fakeRepository, nil) - fakeGit.PlainOpenReturns(&fakeRepository, nil) + fakeGit.PlainOpenReturns(fakeRepository, nil) fakeRepository.TagsReturns(tags, nil) fakeRepository.WorktreeReturns(fakeWorktree, nil) diff --git a/pkg/resourcehandlers/github/github_resource_handler.go b/pkg/resourcehandlers/github/github_resource_handler.go index aa81ca99..c2202f96 100644 --- a/pkg/resourcehandlers/github/github_resource_handler.go +++ b/pkg/resourcehandlers/github/github_resource_handler.go @@ -14,7 +14,6 @@ import ( "strings" "github.com/gardener/docforge/pkg/api" - "github.com/gardener/docforge/pkg/markdown" "github.com/gardener/docforge/pkg/resourcehandlers" "github.com/gardener/docforge/pkg/util/httpclient" "github.com/gardener/docforge/pkg/util/urls" @@ -131,14 +130,15 @@ func (gh *GitHub) buildNodes(ctx context.Context, node *api.Node, excludePaths [ // check for frontMatter filter compliance if frontMatter != nil || excludeFrontMatter != nil { // TODO: cache and reuse to avoid redundant reads when the structure nodes are processed - b, err := gh.Read(ctx, childResourceLocator.String()) - if err != nil { - return nil, err - } - selected, err := markdown.MatchFrontMatterRules(b, frontMatter, excludeFrontMatter) - if err != nil { - return nil, err - } + // TODO: filter nodes once the content is read & parsed + //b, err := gh.Read(ctx, childResourceLocator.String()) + //if err != nil { + // return nil, err + //} + selected := true //, err := markdown.MatchFrontMatterRules(b, frontMatter, excludeFrontMatter) + //if err != nil { + // return nil, err + //} if !selected { continue } diff --git a/pkg/writers/fswriter.go b/pkg/writers/fswriter.go index 62407835..08e55083 100644 --- a/pkg/writers/fswriter.go +++ b/pkg/writers/fswriter.go @@ -5,8 +5,10 @@ package writers import ( + "bytes" "fmt" "github.com/gardener/docforge/pkg/api" + "gopkg.in/yaml.v3" "io/ioutil" "os" "path/filepath" @@ -16,14 +18,38 @@ import ( type FSWriter struct { Root string Ext string + Hugo bool } func (f *FSWriter) Write(name, path string, docBlob []byte, node *api.Node) error { - p := filepath.Join(f.Root, path) - if len(docBlob) <= 0 { - if err := os.MkdirAll(p, os.ModePerm); err != nil { - return err + if f.Hugo && node != nil { + if docBlob == nil && node.Properties != nil && node.Properties["frontmatter"] != nil { + if len(node.Nodes) > 0 { + for _, n := range node.Nodes { + if n.Name == "_index.md" { // TODO: Unify section file check & ensure one section file per folder + // has index child + return nil + } + } + } + // transform params + buf := bytes.Buffer{} + _, _ = buf.Write([]byte("---\n")) + fm, err := yaml.Marshal(node.Properties["frontmatter"]) + if err != nil { + return err + } + _, _ = buf.Write(fm) + _, _ = buf.Write([]byte("---\n")) + docBlob = buf.Bytes() + path = filepath.Join(path, name) + name = "_index.md" } + } + + p := filepath.Join(f.Root, path) + + if len(docBlob) == 0 { return nil } if err := os.MkdirAll(p, os.ModePerm); err != nil { @@ -37,7 +63,7 @@ func (f *FSWriter) Write(name, path string, docBlob []byte, node *api.Node) erro filePath := filepath.Join(p, name) if err := ioutil.WriteFile(filePath, docBlob, 0644); err != nil { - return fmt.Errorf("error writing %s: %v", filepath.Join(f.Root, path, name), err) + return fmt.Errorf("error writing %s: %v", filePath, err) } return nil diff --git a/vendor/github.com/yuin/goldmark-meta/.gitignore b/vendor/github.com/yuin/goldmark-meta/.gitignore new file mode 100644 index 00000000..6e4db922 --- /dev/null +++ b/vendor/github.com/yuin/goldmark-meta/.gitignore @@ -0,0 +1,13 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test +*.pprof + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out diff --git a/vendor/github.com/yuin/goldmark-meta/LICENSE b/vendor/github.com/yuin/goldmark-meta/LICENSE new file mode 100644 index 00000000..dc5b2a69 --- /dev/null +++ b/vendor/github.com/yuin/goldmark-meta/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Yusuke Inuzuka + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/yuin/goldmark-meta/README.md b/vendor/github.com/yuin/goldmark-meta/README.md new file mode 100644 index 00000000..234ce0fb --- /dev/null +++ b/vendor/github.com/yuin/goldmark-meta/README.md @@ -0,0 +1,152 @@ +goldmark-meta +========================= +[![GoDev][godev-image]][godev-url] + +[godev-image]: https://pkg.go.dev/badge/github.com/yuin/goldmark-meta +[godev-url]: https://pkg.go.dev/github.com/yuin/goldmark-meta + + +goldmark-meta is an extension for the [goldmark](http://github.com/yuin/goldmark) +that allows you to define document metadata in YAML format. + +Usage +-------------------- + +### Installation + +``` +go get github.com/yuin/goldmark-meta +``` + +### Markdown syntax + +YAML metadata block is a leaf block that can not have any markdown element +as a child. + +YAML metadata must start with a **YAML metadata separator**. +This separator must be at first line of the document. + +A **YAML metadata separator** is a line that only `-` is repeated. + +YAML metadata must end with a **YAML metadata separator**. + +You can define objects as a 1st level item. At deeper level, you can define +any kind of YAML element. + +Example: + +``` +--- +Title: goldmark-meta +Summary: Add YAML metadata to the document +Tags: + - markdown + - goldmark +--- + +# Heading 1 +``` + + +### Access the metadata + +```go +import ( + "bytes" + "fmt" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark-meta" +) + +func main() { + markdown := goldmark.New( + goldmark.WithExtensions( + meta.Meta, + ), + ) + source := `--- +Title: goldmark-meta +Summary: Add YAML metadata to the document +Tags: + - markdown + - goldmark +--- + +# Hello goldmark-meta +` + + var buf bytes.Buffer + context := parser.NewContext() + if err := markdown.Convert([]byte(source), &buf, parser.WithContext(context)); err != nil { + panic(err) + } + metaData := meta.Get(context) + title := metaData["Title"] + fmt.Print(title) +} +``` + +### Render the metadata as a table + +You need to add `extension.TableHTMLRenderer` or the `Table` extension to +render metadata as a table. + +```go +import ( + "bytes" + "fmt" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/util" + "github.com/yuin/goldmark-meta" +) + +func main() { + markdown := goldmark.New( + goldmark.WithExtensions( + meta.New(meta.WithTable()), + ), + goldmark.WithRendererOptions( + renderer.WithNodeRenderers( + util.Prioritized(extension.NewTableHTMLRenderer(), 500), + ), + ), + ) + // OR + // markdown := goldmark.New( + // goldmark.WithExtensions( + // meta.New(meta.WithTable()), + // extension.Table, + // ), + // ) + source := `--- +Title: goldmark-meta +Summary: Add YAML metadata to the document +Tags: + - markdown + - goldmark +--- + +# Hello goldmark-meta +` + + var buf bytes.Buffer + if err := markdown.Convert([]byte(source), &buf); err != nil { + panic(err) + } + fmt.Print(buf.String()) +} +``` + + +License +-------------------- +MIT + +Author +-------------------- +Yusuke Inuzuka diff --git a/vendor/github.com/yuin/goldmark-meta/meta.go b/vendor/github.com/yuin/goldmark-meta/meta.go new file mode 100644 index 00000000..7850e5cb --- /dev/null +++ b/vendor/github.com/yuin/goldmark-meta/meta.go @@ -0,0 +1,248 @@ +// package meta is a extension for the goldmark(http://github.com/yuin/goldmark). +// +// This extension parses YAML metadata blocks and store metadata to a +// parser.Context. +package meta + +import ( + "bytes" + "fmt" + + "github.com/yuin/goldmark" + gast "github.com/yuin/goldmark/ast" + east "github.com/yuin/goldmark/extension/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" + + "gopkg.in/yaml.v2" +) + +type data struct { + Map map[string]interface{} + Items yaml.MapSlice + Error error + Node gast.Node +} + +var contextKey = parser.NewContextKey() + +// Get returns a YAML metadata. +func Get(pc parser.Context) map[string]interface{} { + v := pc.Get(contextKey) + if v == nil { + return nil + } + d := v.(*data) + return d.Map +} + +// TryGet tries to get a YAML metadata. +// If there are YAML parsing errors, then nil and error are returned +func TryGet(pc parser.Context) (map[string]interface{}, error) { + dtmp := pc.Get(contextKey) + if dtmp == nil { + return nil, nil + } + d := dtmp.(*data) + if d.Error != nil { + return nil, d.Error + } + return d.Map, nil +} + +// GetItems returns a YAML metadata. +// GetItems preserves defined key order. +func GetItems(pc parser.Context) yaml.MapSlice { + v := pc.Get(contextKey) + if v == nil { + return nil + } + d := v.(*data) + return d.Items +} + +// TryGetItems returns a YAML metadata. +// TryGetItems preserves defined key order. +// If there are YAML parsing errors, then nil and erro are returned. +func TryGetItems(pc parser.Context) (yaml.MapSlice, error) { + dtmp := pc.Get(contextKey) + if dtmp == nil { + return nil, nil + } + d := dtmp.(*data) + if d.Error != nil { + return nil, d.Error + } + return d.Items, nil +} + +type metaParser struct { +} + +var defaultMetaParser = &metaParser{} + +// NewParser returns a BlockParser that can parse YAML metadata blocks. +func NewParser() parser.BlockParser { + return defaultMetaParser +} + +func isSeparator(line []byte) bool { + line = util.TrimRightSpace(util.TrimLeftSpace(line)) + for i := 0; i < len(line); i++ { + if line[i] != '-' { + return false + } + } + return true +} + +func (b *metaParser) Trigger() []byte { + return []byte{'-'} +} + +func (b *metaParser) Open(parent gast.Node, reader text.Reader, pc parser.Context) (gast.Node, parser.State) { + linenum, _ := reader.Position() + if linenum != 0 { + return nil, parser.NoChildren + } + line, _ := reader.PeekLine() + if isSeparator(line) { + return gast.NewTextBlock(), parser.NoChildren + } + return nil, parser.NoChildren +} + +func (b *metaParser) Continue(node gast.Node, reader text.Reader, pc parser.Context) parser.State { + line, segment := reader.PeekLine() + if isSeparator(line) && !util.IsBlank(line) { + reader.Advance(segment.Len()) + return parser.Close + } + node.Lines().Append(segment) + return parser.Continue | parser.NoChildren +} + +func (b *metaParser) Close(node gast.Node, reader text.Reader, pc parser.Context) { + lines := node.Lines() + var buf bytes.Buffer + for i := 0; i < lines.Len(); i++ { + segment := lines.At(i) + buf.Write(segment.Value(reader.Source())) + } + d := &data{} + d.Node = node + meta := map[string]interface{}{} + if err := yaml.Unmarshal(buf.Bytes(), &meta); err != nil { + d.Error = err + } else { + d.Map = meta + } + + metaMapSlice := yaml.MapSlice{} + if err := yaml.Unmarshal(buf.Bytes(), &metaMapSlice); err != nil { + d.Error = err + } else { + d.Items = metaMapSlice + } + + pc.Set(contextKey, d) + + if d.Error == nil { + node.Parent().RemoveChild(node.Parent(), node) + } +} + +func (b *metaParser) CanInterruptParagraph() bool { + return false +} + +func (b *metaParser) CanAcceptIndentedLine() bool { + return false +} + +type astTransformer struct { +} + +var defaultASTTransformer = &astTransformer{} + +func (a *astTransformer) Transform(node *gast.Document, reader text.Reader, pc parser.Context) { + dtmp := pc.Get(contextKey) + if dtmp == nil { + return + } + d := dtmp.(*data) + if d.Error != nil { + msg := gast.NewString([]byte(fmt.Sprintf("<!-- %s -->", d.Error))) + msg.SetCode(true) + d.Node.AppendChild(d.Node, msg) + return + } + + meta := GetItems(pc) + if meta == nil { + return + } + table := east.NewTable() + alignments := []east.Alignment{} + for range meta { + alignments = append(alignments, east.AlignNone) + } + row := east.NewTableRow(alignments) + for _, item := range meta { + cell := east.NewTableCell() + cell.AppendChild(cell, gast.NewString([]byte(fmt.Sprintf("%v", item.Key)))) + row.AppendChild(row, cell) + } + table.AppendChild(table, east.NewTableHeader(row)) + + row = east.NewTableRow(alignments) + for _, item := range meta { + cell := east.NewTableCell() + cell.AppendChild(cell, gast.NewString([]byte(fmt.Sprintf("%v", item.Value)))) + row.AppendChild(row, cell) + } + table.AppendChild(table, row) + node.InsertBefore(node, node.FirstChild(), table) +} + +// Option is a functional option type for this extension. +type Option func(*meta) + +// WithTable is a functional option that renders a YAML metadata as a table. +func WithTable() Option { + return func(m *meta) { + m.Table = true + } +} + +type meta struct { + Table bool +} + +// Meta is a extension for the goldmark. +var Meta = &meta{} + +// New returns a new Meta extension. +func New(opts ...Option) goldmark.Extender { + e := &meta{} + for _, opt := range opts { + opt(e) + } + return e +} + +func (e *meta) Extend(m goldmark.Markdown) { + m.Parser().AddOptions( + parser.WithBlockParsers( + util.Prioritized(NewParser(), 0), + ), + ) + if e.Table { + m.Parser().AddOptions( + parser.WithASTTransformers( + util.Prioritized(defaultASTTransformer, 0), + ), + ) + } +} diff --git a/vendor/github.com/yuin/goldmark/.gitignore b/vendor/github.com/yuin/goldmark/.gitignore new file mode 100644 index 00000000..06c135f1 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/.gitignore @@ -0,0 +1,19 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test +*.pprof + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +.DS_Store +fuzz/corpus +fuzz/crashers +fuzz/suppressions +fuzz/fuzz-fuzz.zip diff --git a/vendor/github.com/yuin/goldmark/LICENSE b/vendor/github.com/yuin/goldmark/LICENSE new file mode 100644 index 00000000..dc5b2a69 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Yusuke Inuzuka + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/yuin/goldmark/Makefile b/vendor/github.com/yuin/goldmark/Makefile new file mode 100644 index 00000000..667a19a4 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/Makefile @@ -0,0 +1,16 @@ +.PHONY: test fuzz + +test: + go test -coverprofile=profile.out -coverpkg=github.com/yuin/goldmark,github.com/yuin/goldmark/ast,github.com/yuin/goldmark/extension,github.com/yuin/goldmark/extension/ast,github.com/yuin/goldmark/parser,github.com/yuin/goldmark/renderer,github.com/yuin/goldmark/renderer/html,github.com/yuin/goldmark/text,github.com/yuin/goldmark/util ./... + +cov: test + go tool cover -html=profile.out + +fuzz: + which go-fuzz > /dev/null 2>&1 || (GO111MODULE=off go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build; GO111MODULE=off go get -d github.com/dvyukov/go-fuzz-corpus; true) + rm -rf ./fuzz/corpus + rm -rf ./fuzz/crashers + rm -rf ./fuzz/suppressions + rm -f ./fuzz/fuzz-fuzz.zip + cd ./fuzz && go-fuzz-build + cd ./fuzz && go-fuzz diff --git a/vendor/github.com/yuin/goldmark/README.md b/vendor/github.com/yuin/goldmark/README.md new file mode 100644 index 00000000..2e54d9b8 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/README.md @@ -0,0 +1,494 @@ +goldmark +========================================== + +[![https://pkg.go.dev/github.com/yuin/goldmark](https://pkg.go.dev/badge/github.com/yuin/goldmark.svg)](https://pkg.go.dev/github.com/yuin/goldmark) +[![https://github.com/yuin/goldmark/actions?query=workflow:test](https://github.com/yuin/goldmark/workflows/test/badge.svg?branch=master&event=push)](https://github.com/yuin/goldmark/actions?query=workflow:test) +[![https://coveralls.io/github/yuin/goldmark](https://coveralls.io/repos/github/yuin/goldmark/badge.svg?branch=master)](https://coveralls.io/github/yuin/goldmark) +[![https://goreportcard.com/report/github.com/yuin/goldmark](https://goreportcard.com/badge/github.com/yuin/goldmark)](https://goreportcard.com/report/github.com/yuin/goldmark) + +> A Markdown parser written in Go. Easy to extend, standards-compliant, well-structured. + +goldmark is compliant with CommonMark 0.30. + +Motivation +---------------------- +I needed a Markdown parser for Go that satisfies the following requirements: + +- Easy to extend. + - Markdown is poor in document expressions compared to other light markup languages such as reStructuredText. + - We have extensions to the Markdown syntax, e.g. PHP Markdown Extra, GitHub Flavored Markdown. +- Standards-compliant. + - Markdown has many dialects. + - GitHub-Flavored Markdown is widely used and is based upon CommonMark, effectively mooting the question of whether or not CommonMark is an ideal specification. + - CommonMark is complicated and hard to implement. +- Well-structured. + - AST-based; preserves source position of nodes. +- Written in pure Go. + +[golang-commonmark](https://gitlab.com/golang-commonmark/markdown) may be a good choice, but it seems to be a copy of [markdown-it](https://github.com/markdown-it). + +[blackfriday.v2](https://github.com/russross/blackfriday/tree/v2) is a fast and widely-used implementation, but is not CommonMark-compliant and cannot be extended from outside of the package, since its AST uses structs instead of interfaces. + +Furthermore, its behavior differs from other implementations in some cases, especially regarding lists: [Deep nested lists don't output correctly #329](https://github.com/russross/blackfriday/issues/329), [List block cannot have a second line #244](https://github.com/russross/blackfriday/issues/244), etc. + +This behavior sometimes causes problems. If you migrate your Markdown text from GitHub to blackfriday-based wikis, many lists will immediately be broken. + +As mentioned above, CommonMark is complicated and hard to implement, so Markdown parsers based on CommonMark are few and far between. + +Features +---------------------- + +- **Standards-compliant.** goldmark is fully compliant with the latest [CommonMark](https://commonmark.org/) specification. +- **Extensible.** Do you want to add a `@username` mention syntax to Markdown? + You can easily do so in goldmark. You can add your AST nodes, + parsers for block-level elements, parsers for inline-level elements, + transformers for paragraphs, transformers for the whole AST structure, and + renderers. +- **Performance.** goldmark's performance is on par with that of cmark, + the CommonMark reference implementation written in C. +- **Robust.** goldmark is tested with [go-fuzz](https://github.com/dvyukov/go-fuzz), a fuzz testing tool. +- **Built-in extensions.** goldmark ships with common extensions like tables, strikethrough, + task lists, and definition lists. +- **Depends only on standard libraries.** + +Installation +---------------------- +```bash +$ go get github.com/yuin/goldmark +``` + + +Usage +---------------------- +Import packages: + +```go +import ( + "bytes" + "github.com/yuin/goldmark" +) +``` + + +Convert Markdown documents with the CommonMark-compliant mode: + +```go +var buf bytes.Buffer +if err := goldmark.Convert(source, &buf); err != nil { + panic(err) +} +``` + +With options +------------------------------ + +```go +var buf bytes.Buffer +if err := goldmark.Convert(source, &buf, parser.WithContext(ctx)); err != nil { + panic(err) +} +``` + +| Functional option | Type | Description | +| ----------------- | ---- | ----------- | +| `parser.WithContext` | A `parser.Context` | Context for the parsing phase. | + +Context options +---------------------- + +| Functional option | Type | Description | +| ----------------- | ---- | ----------- | +| `parser.WithIDs` | A `parser.IDs` | `IDs` allows you to change logics that are related to element id(ex: Auto heading id generation). | + + +Custom parser and renderer +-------------------------- +```go +import ( + "bytes" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer/html" +) + +md := goldmark.New( + goldmark.WithExtensions(extension.GFM), + goldmark.WithParserOptions( + parser.WithAutoHeadingID(), + ), + goldmark.WithRendererOptions( + html.WithHardWraps(), + html.WithXHTML(), + ), + ) +var buf bytes.Buffer +if err := md.Convert(source, &buf); err != nil { + panic(err) +} +``` + +| Functional option | Type | Description | +| ----------------- | ---- | ----------- | +| `goldmark.WithParser` | `parser.Parser` | This option must be passed before `goldmark.WithParserOptions` and `goldmark.WithExtensions` | +| `goldmark.WithRenderer` | `renderer.Renderer` | This option must be passed before `goldmark.WithRendererOptions` and `goldmark.WithExtensions` | +| `goldmark.WithParserOptions` | `...parser.Option` | | +| `goldmark.WithRendererOptions` | `...renderer.Option` | | +| `goldmark.WithExtensions` | `...goldmark.Extender` | | + +Parser and Renderer options +------------------------------ + +### Parser options + +| Functional option | Type | Description | +| ----------------- | ---- | ----------- | +| `parser.WithBlockParsers` | A `util.PrioritizedSlice` whose elements are `parser.BlockParser` | Parsers for parsing block level elements. | +| `parser.WithInlineParsers` | A `util.PrioritizedSlice` whose elements are `parser.InlineParser` | Parsers for parsing inline level elements. | +| `parser.WithParagraphTransformers` | A `util.PrioritizedSlice` whose elements are `parser.ParagraphTransformer` | Transformers for transforming paragraph nodes. | +| `parser.WithASTTransformers` | A `util.PrioritizedSlice` whose elements are `parser.ASTTransformer` | Transformers for transforming an AST. | +| `parser.WithAutoHeadingID` | `-` | Enables auto heading ids. | +| `parser.WithAttribute` | `-` | Enables custom attributes. Currently only headings supports attributes. | + +### HTML Renderer options + +| Functional option | Type | Description | +| ----------------- | ---- | ----------- | +| `html.WithWriter` | `html.Writer` | `html.Writer` for writing contents to an `io.Writer`. | +| `html.WithHardWraps` | `-` | Render newlines as `<br>`.| +| `html.WithXHTML` | `-` | Render as XHTML. | +| `html.WithUnsafe` | `-` | By default, goldmark does not render raw HTML or potentially dangerous links. With this option, goldmark renders such content as written. | + +### Built-in extensions + +- `extension.Table` + - [GitHub Flavored Markdown: Tables](https://github.github.com/gfm/#tables-extension-) +- `extension.Strikethrough` + - [GitHub Flavored Markdown: Strikethrough](https://github.github.com/gfm/#strikethrough-extension-) +- `extension.Linkify` + - [GitHub Flavored Markdown: Autolinks](https://github.github.com/gfm/#autolinks-extension-) +- `extension.TaskList` + - [GitHub Flavored Markdown: Task list items](https://github.github.com/gfm/#task-list-items-extension-) +- `extension.GFM` + - This extension enables Table, Strikethrough, Linkify and TaskList. + - This extension does not filter tags defined in [6.11: Disallowed Raw HTML (extension)](https://github.github.com/gfm/#disallowed-raw-html-extension-). + If you need to filter HTML tags, see [Security](#security). + - If you need to parse github emojis, you can use [goldmark-emoji](https://github.com/yuin/goldmark-emoji) extension. +- `extension.DefinitionList` + - [PHP Markdown Extra: Definition lists](https://michelf.ca/projects/php-markdown/extra/#def-list) +- `extension.Footnote` + - [PHP Markdown Extra: Footnotes](https://michelf.ca/projects/php-markdown/extra/#footnotes) +- `extension.Typographer` + - This extension substitutes punctuations with typographic entities like [smartypants](https://daringfireball.net/projects/smartypants/). + +### Attributes +The `parser.WithAttribute` option allows you to define attributes on some elements. + +Currently only headings support attributes. + +**Attributes are being discussed in the +[CommonMark forum](https://talk.commonmark.org/t/consistent-attribute-syntax/272). +This syntax may possibly change in the future.** + + +#### Headings + +``` +## heading ## {#id .className attrName=attrValue class="class1 class2"} + +## heading {#id .className attrName=attrValue class="class1 class2"} +``` + +``` +heading {#id .className attrName=attrValue} +============ +``` + +### Table extension +The Table extension implements [Table(extension)](https://github.github.com/gfm/#tables-extension-), as +defined in [GitHub Flavored Markdown Spec](https://github.github.com/gfm/). + +Specs are defined for XHTML, so specs use some deprecated attributes for HTML5. + +You can override alignment rendering method via options. + +| Functional option | Type | Description | +| ----------------- | ---- | ----------- | +| `extension.WithTableCellAlignMethod` | `extension.TableCellAlignMethod` | Option indicates how are table cells aligned. | + +### Typographer extension + +The Typographer extension translates plain ASCII punctuation characters into typographic-punctuation HTML entities. + +Default substitutions are: + +| Punctuation | Default entity | +| ------------ | ---------- | +| `'` | `‘`, `’` | +| `"` | `“`, `”` | +| `--` | `–` | +| `---` | `—` | +| `...` | `…` | +| `<<` | `«` | +| `>>` | `»` | + +You can override the default substitutions via `extensions.WithTypographicSubstitutions`: + +```go +markdown := goldmark.New( + goldmark.WithExtensions( + extension.NewTypographer( + extension.WithTypographicSubstitutions(extension.TypographicSubstitutions{ + extension.LeftSingleQuote: []byte("‚"), + extension.RightSingleQuote: nil, // nil disables a substitution + }), + ), + ), +) +``` + +### Linkify extension + +The Linkify extension implements [Autolinks(extension)](https://github.github.com/gfm/#autolinks-extension-), as +defined in [GitHub Flavored Markdown Spec](https://github.github.com/gfm/). + +Since the spec does not define details about URLs, there are numerous ambiguous cases. + +You can override autolinking patterns via options. + +| Functional option | Type | Description | +| ----------------- | ---- | ----------- | +| `extension.WithLinkifyAllowedProtocols` | `[][]byte` | List of allowed protocols such as `[][]byte{ []byte("http:") }` | +| `extension.WithLinkifyURLRegexp` | `*regexp.Regexp` | Regexp that defines URLs, including protocols | +| `extension.WithLinkifyWWWRegexp` | `*regexp.Regexp` | Regexp that defines URL starting with `www.`. This pattern corresponds to [the extended www autolink](https://github.github.com/gfm/#extended-www-autolink) | +| `extension.WithLinkifyEmailRegexp` | `*regexp.Regexp` | Regexp that defines email addresses` | + +Example, using [xurls](https://github.com/mvdan/xurls): + +```go +import "mvdan.cc/xurls/v2" + +markdown := goldmark.New( + goldmark.WithRendererOptions( + html.WithXHTML(), + html.WithUnsafe(), + ), + goldmark.WithExtensions( + extension.NewLinkify( + extension.WithLinkifyAllowedProtocols([][]byte{ + []byte("http:"), + []byte("https:"), + }), + extension.WithLinkifyURLRegexp( + xurls.Strict, + ), + ), + ), +) +``` + +### Footnotes extension + +The Footnote extension implements [PHP Markdown Extra: Footnotes](https://michelf.ca/projects/php-markdown/extra/#footnotes). + +This extension has some options: + +| Functional option | Type | Description | +| ----------------- | ---- | ----------- | +| `extension.WithFootnoteIDPrefix` | `[]byte` | a prefix for the id attributes.| +| `extension.WithFootnoteIDPrefixFunction` | `func(gast.Node) []byte` | a function that determines the id attribute for given Node.| +| `extension.WithFootnoteLinkTitle` | `[]byte` | an optional title attribute for footnote links.| +| `extension.WithFootnoteBacklinkTitle` | `[]byte` | an optional title attribute for footnote backlinks. | +| `extension.WithFootnoteLinkClass` | `[]byte` | a class for footnote links. This defaults to `footnote-ref`. | +| `extension.WithFootnoteBacklinkClass` | `[]byte` | a class for footnote backlinks. This defaults to `footnote-backref`. | +| `extension.WithFootnoteBacklinkHTML` | `[]byte` | a class for footnote backlinks. This defaults to `↩︎`. | + +Some options can have special substitutions. Occurrences of “^^” in the string will be replaced by the corresponding footnote number in the HTML output. Occurrences of “%%” will be replaced by a number for the reference (footnotes can have multiple references). + +`extension.WithFootnoteIDPrefix` and `extension.WithFootnoteIDPrefixFunction` are useful if you have multiple Markdown documents displayed inside one HTML document to avoid footnote ids to clash each other. + +`extension.WithFootnoteIDPrefix` sets fixed id prefix, so you may write codes like the following: + +```go +for _, path := range files { + source := readAll(path) + prefix := getPrefix(path) + + markdown := goldmark.New( + goldmark.WithExtensions( + NewFootnote( + WithFootnoteIDPrefix([]byte(path)), + ), + ), + ) + var b bytes.Buffer + err := markdown.Convert(source, &b) + if err != nil { + t.Error(err.Error()) + } +} +``` + +`extension.WithFootnoteIDPrefixFunction` determines an id prefix by calling given function, so you may write codes like the following: + +```go +markdown := goldmark.New( + goldmark.WithExtensions( + NewFootnote( + WithFootnoteIDPrefixFunction(func(n gast.Node) []byte { + v, ok := n.OwnerDocument().Meta()["footnote-prefix"] + if ok { + return util.StringToReadOnlyBytes(v.(string)) + } + return nil + }), + ), + ), +) + +for _, path := range files { + source := readAll(path) + var b bytes.Buffer + + doc := markdown.Parser().Parse(text.NewReader(source)) + doc.Meta()["footnote-prefix"] = getPrefix(path) + err := markdown.Renderer().Render(&b, source, doc) +} +``` + +You can use [goldmark-meta](https://github.com/yuin/goldmark-meta) to define a id prefix in the markdown document: + + +```markdown +--- +title: document title +slug: article1 +footnote-prefix: article1 +--- + +# My article + +``` + +Security +-------------------- +By default, goldmark does not render raw HTML or potentially-dangerous URLs. +If you need to gain more control over untrusted contents, it is recommended that you +use an HTML sanitizer such as [bluemonday](https://github.com/microcosm-cc/bluemonday). + +Benchmark +-------------------- +You can run this benchmark in the `_benchmark` directory. + +### against other golang libraries + +blackfriday v2 seems to be the fastest, but as it is not CommonMark compliant, its performance cannot be directly compared to that of the CommonMark-compliant libraries. + +goldmark, meanwhile, builds a clean, extensible AST structure, achieves full compliance with +CommonMark, and consumes less memory, all while being reasonably fast. + +``` +goos: darwin +goarch: amd64 +BenchmarkMarkdown/Blackfriday-v2-12 326 3465240 ns/op 3298861 B/op 20047 allocs/op +BenchmarkMarkdown/GoldMark-12 303 3927494 ns/op 2574809 B/op 13853 allocs/op +BenchmarkMarkdown/CommonMark-12 244 4900853 ns/op 2753851 B/op 20527 allocs/op +BenchmarkMarkdown/Lute-12 130 9195245 ns/op 9175030 B/op 123534 allocs/op +BenchmarkMarkdown/GoMarkdown-12 9 113541994 ns/op 2187472 B/op 22173 allocs/op +``` + +### against cmark (CommonMark reference implementation written in C) + +``` +----------- cmark ----------- +file: _data.md +iteration: 50 +average: 0.0037760639 sec +go run ./goldmark_benchmark.go +------- goldmark ------- +file: _data.md +iteration: 50 +average: 0.0040964230 sec +``` + +As you can see, goldmark's performance is on par with cmark's. + +Extensions +-------------------- + +- [goldmark-meta](https://github.com/yuin/goldmark-meta): A YAML metadata + extension for the goldmark Markdown parser. +- [goldmark-highlighting](https://github.com/yuin/goldmark-highlighting): A syntax-highlighting extension + for the goldmark markdown parser. +- [goldmark-emoji](https://github.com/yuin/goldmark-emoji): An emoji + extension for the goldmark Markdown parser. +- [goldmark-mathjax](https://github.com/litao91/goldmark-mathjax): Mathjax support for the goldmark markdown parser +- [goldmark-pdf](https://github.com/stephenafamo/goldmark-pdf): A PDF renderer that can be passed to `goldmark.WithRenderer()`. +- [goldmark-hashtag](https://github.com/abhinav/goldmark-hashtag): Adds support for `#hashtag`-based tagging to goldmark. +- [goldmark-wikilink](https://github.com/abhinav/goldmark-wikilink): Adds support for `[[wiki]]`-style links to goldmark. +- [goldmark-toc](https://github.com/abhinav/goldmark-toc): Adds support for generating tables-of-contents for goldmark documents. +- [goldmark-mermaid](https://github.com/abhinav/goldmark-mermaid): Adds support for renderng [Mermaid](https://mermaid-js.github.io/mermaid/) diagrams in goldmark documents. + +goldmark internal(for extension developers) +---------------------------------------------- +### Overview +goldmark's Markdown processing is outlined in the diagram below. + +``` + <Markdown in []byte, parser.Context> + | + V + +-------- parser.Parser --------------------------- + | 1. Parse block elements into AST + | 1. If a parsed block is a paragraph, apply + | ast.ParagraphTransformer + | 2. Traverse AST and parse blocks. + | 1. Process delimiters(emphasis) at the end of + | block parsing + | 3. Apply parser.ASTTransformers to AST + | + V + <ast.Node> + | + V + +------- renderer.Renderer ------------------------ + | 1. Traverse AST and apply renderer.NodeRenderer + | corespond to the node type + + | + V + <Output> +``` + +### Parsing +Markdown documents are read through `text.Reader` interface. + +AST nodes do not have concrete text. AST nodes have segment information of the documents, represented by `text.Segment` . + +`text.Segment` has 3 attributes: `Start`, `End`, `Padding` . + +(TBC) + +**TODO** + +See `extension` directory for examples of extensions. + +Summary: + +1. Define AST Node as a struct in which `ast.BaseBlock` or `ast.BaseInline` is embedded. +2. Write a parser that implements `parser.BlockParser` or `parser.InlineParser`. +3. Write a renderer that implements `renderer.NodeRenderer`. +4. Define your goldmark extension that implements `goldmark.Extender`. + + +Donation +-------------------- +BTC: 1NEDSyUmo4SMTDP83JJQSWi1MvQUGGNMZB + +License +-------------------- +MIT + +Author +-------------------- +Yusuke Inuzuka diff --git a/vendor/github.com/yuin/goldmark/ast/ast.go b/vendor/github.com/yuin/goldmark/ast/ast.go new file mode 100644 index 00000000..3719ebbd --- /dev/null +++ b/vendor/github.com/yuin/goldmark/ast/ast.go @@ -0,0 +1,508 @@ +// Package ast defines AST nodes that represent markdown elements. +package ast + +import ( + "bytes" + "fmt" + "strings" + + textm "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +// A NodeType indicates what type a node belongs to. +type NodeType int + +const ( + // TypeBlock indicates that a node is kind of block nodes. + TypeBlock NodeType = iota + 1 + // TypeInline indicates that a node is kind of inline nodes. + TypeInline + // TypeDocument indicates that a node is kind of document nodes. + TypeDocument +) + +// NodeKind indicates more specific type than NodeType. +type NodeKind int + +func (k NodeKind) String() string { + return kindNames[k] +} + +var kindMax NodeKind +var kindNames = []string{""} + +// NewNodeKind returns a new Kind value. +func NewNodeKind(name string) NodeKind { + kindMax++ + kindNames = append(kindNames, name) + return kindMax +} + +// An Attribute is an attribute of the Node +type Attribute struct { + Name []byte + Value interface{} +} + +// A Node interface defines basic AST node functionalities. +type Node interface { + // Type returns a type of this node. + Type() NodeType + + // Kind returns a kind of this node. + Kind() NodeKind + + // NextSibling returns a next sibling node of this node. + NextSibling() Node + + // PreviousSibling returns a previous sibling node of this node. + PreviousSibling() Node + + // Parent returns a parent node of this node. + Parent() Node + + // SetParent sets a parent node to this node. + SetParent(Node) + + // SetPreviousSibling sets a previous sibling node to this node. + SetPreviousSibling(Node) + + // SetNextSibling sets a next sibling node to this node. + SetNextSibling(Node) + + // HasChildren returns true if this node has any children, otherwise false. + HasChildren() bool + + // ChildCount returns a total number of children. + ChildCount() int + + // FirstChild returns a first child of this node. + FirstChild() Node + + // LastChild returns a last child of this node. + LastChild() Node + + // AppendChild append a node child to the tail of the children. + AppendChild(self, child Node) + + // RemoveChild removes a node child from this node. + // If a node child is not children of this node, RemoveChild nothing to do. + RemoveChild(self, child Node) + + // RemoveChildren removes all children from this node. + RemoveChildren(self Node) + + // SortChildren sorts childrens by comparator. + SortChildren(comparator func(n1, n2 Node) int) + + // ReplaceChild replace a node v1 with a node insertee. + // If v1 is not children of this node, ReplaceChild append a insetee to the + // tail of the children. + ReplaceChild(self, v1, insertee Node) + + // InsertBefore inserts a node insertee before a node v1. + // If v1 is not children of this node, InsertBefore append a insetee to the + // tail of the children. + InsertBefore(self, v1, insertee Node) + + // InsertAfterinserts a node insertee after a node v1. + // If v1 is not children of this node, InsertBefore append a insetee to the + // tail of the children. + InsertAfter(self, v1, insertee Node) + + // OwnerDocument returns this node's owner document. + // If this node is not a child of the Document node, OwnerDocument + // returns nil. + OwnerDocument() *Document + + // Dump dumps an AST tree structure to stdout. + // This function completely aimed for debugging. + // level is a indent level. Implementer should indent informations with + // 2 * level spaces. + Dump(source []byte, level int) + + // Text returns text values of this node. + Text(source []byte) []byte + + // HasBlankPreviousLines returns true if the row before this node is blank, + // otherwise false. + // This method is valid only for block nodes. + HasBlankPreviousLines() bool + + // SetBlankPreviousLines sets whether the row before this node is blank. + // This method is valid only for block nodes. + SetBlankPreviousLines(v bool) + + // Lines returns text segments that hold positions in a source. + // This method is valid only for block nodes. + Lines() *textm.Segments + + // SetLines sets text segments that hold positions in a source. + // This method is valid only for block nodes. + SetLines(*textm.Segments) + + // IsRaw returns true if contents should be rendered as 'raw' contents. + IsRaw() bool + + // SetAttribute sets the given value to the attributes. + SetAttribute(name []byte, value interface{}) + + // SetAttributeString sets the given value to the attributes. + SetAttributeString(name string, value interface{}) + + // Attribute returns a (attribute value, true) if an attribute + // associated with the given name is found, otherwise + // (nil, false) + Attribute(name []byte) (interface{}, bool) + + // AttributeString returns a (attribute value, true) if an attribute + // associated with the given name is found, otherwise + // (nil, false) + AttributeString(name string) (interface{}, bool) + + // Attributes returns a list of attributes. + // This may be a nil if there are no attributes. + Attributes() []Attribute + + // RemoveAttributes removes all attributes from this node. + RemoveAttributes() +} + +// A BaseNode struct implements the Node interface partialliy. +type BaseNode struct { + firstChild Node + lastChild Node + parent Node + next Node + prev Node + childCount int + attributes []Attribute +} + +func ensureIsolated(v Node) { + if p := v.Parent(); p != nil { + p.RemoveChild(p, v) + } +} + +// HasChildren implements Node.HasChildren . +func (n *BaseNode) HasChildren() bool { + return n.firstChild != nil +} + +// SetPreviousSibling implements Node.SetPreviousSibling . +func (n *BaseNode) SetPreviousSibling(v Node) { + n.prev = v +} + +// SetNextSibling implements Node.SetNextSibling . +func (n *BaseNode) SetNextSibling(v Node) { + n.next = v +} + +// PreviousSibling implements Node.PreviousSibling . +func (n *BaseNode) PreviousSibling() Node { + return n.prev +} + +// NextSibling implements Node.NextSibling . +func (n *BaseNode) NextSibling() Node { + return n.next +} + +// RemoveChild implements Node.RemoveChild . +func (n *BaseNode) RemoveChild(self, v Node) { + if v.Parent() != self { + return + } + n.childCount-- + prev := v.PreviousSibling() + next := v.NextSibling() + if prev != nil { + prev.SetNextSibling(next) + } else { + n.firstChild = next + } + if next != nil { + next.SetPreviousSibling(prev) + } else { + n.lastChild = prev + } + v.SetParent(nil) + v.SetPreviousSibling(nil) + v.SetNextSibling(nil) +} + +// RemoveChildren implements Node.RemoveChildren . +func (n *BaseNode) RemoveChildren(self Node) { + for c := n.firstChild; c != nil; { + c.SetParent(nil) + c.SetPreviousSibling(nil) + next := c.NextSibling() + c.SetNextSibling(nil) + c = next + } + n.firstChild = nil + n.lastChild = nil + n.childCount = 0 +} + +// SortChildren implements Node.SortChildren +func (n *BaseNode) SortChildren(comparator func(n1, n2 Node) int) { + var sorted Node + current := n.firstChild + for current != nil { + next := current.NextSibling() + if sorted == nil || comparator(sorted, current) >= 0 { + current.SetNextSibling(sorted) + if sorted != nil { + sorted.SetPreviousSibling(current) + } + sorted = current + sorted.SetPreviousSibling(nil) + } else { + c := sorted + for c.NextSibling() != nil && comparator(c.NextSibling(), current) < 0 { + c = c.NextSibling() + } + current.SetNextSibling(c.NextSibling()) + current.SetPreviousSibling(c) + if c.NextSibling() != nil { + c.NextSibling().SetPreviousSibling(current) + } + c.SetNextSibling(current) + } + current = next + } + n.firstChild = sorted + for c := n.firstChild; c != nil; c = c.NextSibling() { + n.lastChild = c + } +} + +// FirstChild implements Node.FirstChild . +func (n *BaseNode) FirstChild() Node { + return n.firstChild +} + +// LastChild implements Node.LastChild . +func (n *BaseNode) LastChild() Node { + return n.lastChild +} + +// ChildCount implements Node.ChildCount . +func (n *BaseNode) ChildCount() int { + return n.childCount +} + +// Parent implements Node.Parent . +func (n *BaseNode) Parent() Node { + return n.parent +} + +// SetParent implements Node.SetParent . +func (n *BaseNode) SetParent(v Node) { + n.parent = v +} + +// AppendChild implements Node.AppendChild . +func (n *BaseNode) AppendChild(self, v Node) { + ensureIsolated(v) + if n.firstChild == nil { + n.firstChild = v + v.SetNextSibling(nil) + v.SetPreviousSibling(nil) + } else { + last := n.lastChild + last.SetNextSibling(v) + v.SetPreviousSibling(last) + } + v.SetParent(self) + n.lastChild = v + n.childCount++ +} + +// ReplaceChild implements Node.ReplaceChild . +func (n *BaseNode) ReplaceChild(self, v1, insertee Node) { + n.InsertBefore(self, v1, insertee) + n.RemoveChild(self, v1) +} + +// InsertAfter implements Node.InsertAfter . +func (n *BaseNode) InsertAfter(self, v1, insertee Node) { + n.InsertBefore(self, v1.NextSibling(), insertee) +} + +// InsertBefore implements Node.InsertBefore . +func (n *BaseNode) InsertBefore(self, v1, insertee Node) { + n.childCount++ + if v1 == nil { + n.AppendChild(self, insertee) + return + } + ensureIsolated(insertee) + if v1.Parent() == self { + c := v1 + prev := c.PreviousSibling() + if prev != nil { + prev.SetNextSibling(insertee) + insertee.SetPreviousSibling(prev) + } else { + n.firstChild = insertee + insertee.SetPreviousSibling(nil) + } + insertee.SetNextSibling(c) + c.SetPreviousSibling(insertee) + insertee.SetParent(self) + } +} + +// OwnerDocument implements Node.OwnerDocument +func (n *BaseNode) OwnerDocument() *Document { + d := n.Parent() + for { + p := d.Parent() + if p == nil { + if v, ok := d.(*Document); ok { + return v + } + break + } + d = p + } + return nil +} + +// Text implements Node.Text . +func (n *BaseNode) Text(source []byte) []byte { + var buf bytes.Buffer + for c := n.firstChild; c != nil; c = c.NextSibling() { + buf.Write(c.Text(source)) + } + return buf.Bytes() +} + +// SetAttribute implements Node.SetAttribute. +func (n *BaseNode) SetAttribute(name []byte, value interface{}) { + if n.attributes == nil { + n.attributes = make([]Attribute, 0, 10) + } else { + for i, a := range n.attributes { + if bytes.Equal(a.Name, name) { + n.attributes[i].Name = name + n.attributes[i].Value = value + return + } + } + } + n.attributes = append(n.attributes, Attribute{name, value}) +} + +// SetAttributeString implements Node.SetAttributeString +func (n *BaseNode) SetAttributeString(name string, value interface{}) { + n.SetAttribute(util.StringToReadOnlyBytes(name), value) +} + +// Attribute implements Node.Attribute. +func (n *BaseNode) Attribute(name []byte) (interface{}, bool) { + if n.attributes == nil { + return nil, false + } + for i, a := range n.attributes { + if bytes.Equal(a.Name, name) { + return n.attributes[i].Value, true + } + } + return nil, false +} + +// AttributeString implements Node.AttributeString. +func (n *BaseNode) AttributeString(s string) (interface{}, bool) { + return n.Attribute(util.StringToReadOnlyBytes(s)) +} + +// Attributes implements Node.Attributes +func (n *BaseNode) Attributes() []Attribute { + return n.attributes +} + +// RemoveAttributes implements Node.RemoveAttributes +func (n *BaseNode) RemoveAttributes() { + n.attributes = nil +} + +// DumpHelper is a helper function to implement Node.Dump. +// kv is pairs of an attribute name and an attribute value. +// cb is a function called after wrote a name and attributes. +func DumpHelper(v Node, source []byte, level int, kv map[string]string, cb func(int)) { + name := v.Kind().String() + indent := strings.Repeat(" ", level) + fmt.Printf("%s%s {\n", indent, name) + indent2 := strings.Repeat(" ", level+1) + if v.Type() == TypeBlock { + fmt.Printf("%sRawText: \"", indent2) + for i := 0; i < v.Lines().Len(); i++ { + line := v.Lines().At(i) + fmt.Printf("%s", line.Value(source)) + } + fmt.Printf("\"\n") + fmt.Printf("%sHasBlankPreviousLines: %v\n", indent2, v.HasBlankPreviousLines()) + } + for name, value := range kv { + fmt.Printf("%s%s: %s\n", indent2, name, value) + } + if cb != nil { + cb(level + 1) + } + for c := v.FirstChild(); c != nil; c = c.NextSibling() { + c.Dump(source, level+1) + } + fmt.Printf("%s}\n", indent) +} + +// WalkStatus represents a current status of the Walk function. +type WalkStatus int + +const ( + // WalkStop indicates no more walking needed. + WalkStop WalkStatus = iota + 1 + + // WalkSkipChildren indicates that Walk wont walk on children of current + // node. + WalkSkipChildren + + // WalkContinue indicates that Walk can continue to walk. + WalkContinue +) + +// Walker is a function that will be called when Walk find a +// new node. +// entering is set true before walks children, false after walked children. +// If Walker returns error, Walk function immediately stop walking. +type Walker func(n Node, entering bool) (WalkStatus, error) + +// Walk walks a AST tree by the depth first search algorithm. +func Walk(n Node, walker Walker) error { + _, err := walkHelper(n, walker) + return err +} + +func walkHelper(n Node, walker Walker) (WalkStatus, error) { + status, err := walker(n, true) + if err != nil || status == WalkStop { + return status, err + } + if status != WalkSkipChildren { + for c := n.FirstChild(); c != nil; c = c.NextSibling() { + if st, err := walkHelper(c, walker); err != nil || st == WalkStop { + return WalkStop, err + } + } + } + status, err = walker(n, false) + if err != nil || status == WalkStop { + return WalkStop, err + } + return WalkContinue, nil +} diff --git a/vendor/github.com/yuin/goldmark/ast/block.go b/vendor/github.com/yuin/goldmark/ast/block.go new file mode 100644 index 00000000..4a504a81 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/ast/block.go @@ -0,0 +1,495 @@ +package ast + +import ( + "fmt" + "strings" + + textm "github.com/yuin/goldmark/text" +) + +// A BaseBlock struct implements the Node interface partialliy. +type BaseBlock struct { + BaseNode + blankPreviousLines bool + lines *textm.Segments +} + +// Type implements Node.Type +func (b *BaseBlock) Type() NodeType { + return TypeBlock +} + +// IsRaw implements Node.IsRaw +func (b *BaseBlock) IsRaw() bool { + return false +} + +// HasBlankPreviousLines implements Node.HasBlankPreviousLines. +func (b *BaseBlock) HasBlankPreviousLines() bool { + return b.blankPreviousLines +} + +// SetBlankPreviousLines implements Node.SetBlankPreviousLines. +func (b *BaseBlock) SetBlankPreviousLines(v bool) { + b.blankPreviousLines = v +} + +// Lines implements Node.Lines +func (b *BaseBlock) Lines() *textm.Segments { + if b.lines == nil { + b.lines = textm.NewSegments() + } + return b.lines +} + +// SetLines implements Node.SetLines +func (b *BaseBlock) SetLines(v *textm.Segments) { + b.lines = v +} + +// A Document struct is a root node of Markdown text. +type Document struct { + BaseBlock + + meta map[string]interface{} +} + +// KindDocument is a NodeKind of the Document node. +var KindDocument = NewNodeKind("Document") + +// Dump implements Node.Dump . +func (n *Document) Dump(source []byte, level int) { + DumpHelper(n, source, level, nil, nil) +} + +// Type implements Node.Type . +func (n *Document) Type() NodeType { + return TypeDocument +} + +// Kind implements Node.Kind. +func (n *Document) Kind() NodeKind { + return KindDocument +} + +// OwnerDocument implements Node.OwnerDocument +func (n *Document) OwnerDocument() *Document { + return n +} + +// Meta returns metadata of this document. +func (n *Document) Meta() map[string]interface{} { + if n.meta == nil { + n.meta = map[string]interface{}{} + } + return n.meta +} + +// SetMeta sets given metadata to this document. +func (n *Document) SetMeta(meta map[string]interface{}) { + n.meta = meta +} + +// NewDocument returns a new Document node. +func NewDocument() *Document { + return &Document{ + BaseBlock: BaseBlock{}, + meta: nil, + } +} + +// A TextBlock struct is a node whose lines +// should be rendered without any containers. +type TextBlock struct { + BaseBlock +} + +// Dump implements Node.Dump . +func (n *TextBlock) Dump(source []byte, level int) { + DumpHelper(n, source, level, nil, nil) +} + +// KindTextBlock is a NodeKind of the TextBlock node. +var KindTextBlock = NewNodeKind("TextBlock") + +// Kind implements Node.Kind. +func (n *TextBlock) Kind() NodeKind { + return KindTextBlock +} + +// NewTextBlock returns a new TextBlock node. +func NewTextBlock() *TextBlock { + return &TextBlock{ + BaseBlock: BaseBlock{}, + } +} + +// A Paragraph struct represents a paragraph of Markdown text. +type Paragraph struct { + BaseBlock +} + +// Dump implements Node.Dump . +func (n *Paragraph) Dump(source []byte, level int) { + DumpHelper(n, source, level, nil, nil) +} + +// KindParagraph is a NodeKind of the Paragraph node. +var KindParagraph = NewNodeKind("Paragraph") + +// Kind implements Node.Kind. +func (n *Paragraph) Kind() NodeKind { + return KindParagraph +} + +// NewParagraph returns a new Paragraph node. +func NewParagraph() *Paragraph { + return &Paragraph{ + BaseBlock: BaseBlock{}, + } +} + +// IsParagraph returns true if the given node implements the Paragraph interface, +// otherwise false. +func IsParagraph(node Node) bool { + _, ok := node.(*Paragraph) + return ok +} + +// A Heading struct represents headings like SetextHeading and ATXHeading. +type Heading struct { + BaseBlock + // Level returns a level of this heading. + // This value is between 1 and 6. + Level int +} + +// Dump implements Node.Dump . +func (n *Heading) Dump(source []byte, level int) { + m := map[string]string{ + "Level": fmt.Sprintf("%d", n.Level), + } + DumpHelper(n, source, level, m, nil) +} + +// KindHeading is a NodeKind of the Heading node. +var KindHeading = NewNodeKind("Heading") + +// Kind implements Node.Kind. +func (n *Heading) Kind() NodeKind { + return KindHeading +} + +// NewHeading returns a new Heading node. +func NewHeading(level int) *Heading { + return &Heading{ + BaseBlock: BaseBlock{}, + Level: level, + } +} + +// A ThematicBreak struct represents a thematic break of Markdown text. +type ThematicBreak struct { + BaseBlock +} + +// Dump implements Node.Dump . +func (n *ThematicBreak) Dump(source []byte, level int) { + DumpHelper(n, source, level, nil, nil) +} + +// KindThematicBreak is a NodeKind of the ThematicBreak node. +var KindThematicBreak = NewNodeKind("ThematicBreak") + +// Kind implements Node.Kind. +func (n *ThematicBreak) Kind() NodeKind { + return KindThematicBreak +} + +// NewThematicBreak returns a new ThematicBreak node. +func NewThematicBreak() *ThematicBreak { + return &ThematicBreak{ + BaseBlock: BaseBlock{}, + } +} + +// A CodeBlock interface represents an indented code block of Markdown text. +type CodeBlock struct { + BaseBlock +} + +// IsRaw implements Node.IsRaw. +func (n *CodeBlock) IsRaw() bool { + return true +} + +// Dump implements Node.Dump . +func (n *CodeBlock) Dump(source []byte, level int) { + DumpHelper(n, source, level, nil, nil) +} + +// KindCodeBlock is a NodeKind of the CodeBlock node. +var KindCodeBlock = NewNodeKind("CodeBlock") + +// Kind implements Node.Kind. +func (n *CodeBlock) Kind() NodeKind { + return KindCodeBlock +} + +// NewCodeBlock returns a new CodeBlock node. +func NewCodeBlock() *CodeBlock { + return &CodeBlock{ + BaseBlock: BaseBlock{}, + } +} + +// A FencedCodeBlock struct represents a fenced code block of Markdown text. +type FencedCodeBlock struct { + BaseBlock + // Info returns a info text of this fenced code block. + Info *Text + + language []byte +} + +// Language returns an language in an info string. +// Language returns nil if this node does not have an info string. +func (n *FencedCodeBlock) Language(source []byte) []byte { + if n.language == nil && n.Info != nil { + segment := n.Info.Segment + info := segment.Value(source) + i := 0 + for ; i < len(info); i++ { + if info[i] == ' ' { + break + } + } + n.language = info[:i] + } + return n.language +} + +// IsRaw implements Node.IsRaw. +func (n *FencedCodeBlock) IsRaw() bool { + return true +} + +// Dump implements Node.Dump . +func (n *FencedCodeBlock) Dump(source []byte, level int) { + m := map[string]string{} + if n.Info != nil { + m["Info"] = fmt.Sprintf("\"%s\"", n.Info.Text(source)) + } + DumpHelper(n, source, level, m, nil) +} + +// KindFencedCodeBlock is a NodeKind of the FencedCodeBlock node. +var KindFencedCodeBlock = NewNodeKind("FencedCodeBlock") + +// Kind implements Node.Kind. +func (n *FencedCodeBlock) Kind() NodeKind { + return KindFencedCodeBlock +} + +// NewFencedCodeBlock return a new FencedCodeBlock node. +func NewFencedCodeBlock(info *Text) *FencedCodeBlock { + return &FencedCodeBlock{ + BaseBlock: BaseBlock{}, + Info: info, + } +} + +// A Blockquote struct represents an blockquote block of Markdown text. +type Blockquote struct { + BaseBlock +} + +// Dump implements Node.Dump . +func (n *Blockquote) Dump(source []byte, level int) { + DumpHelper(n, source, level, nil, nil) +} + +// KindBlockquote is a NodeKind of the Blockquote node. +var KindBlockquote = NewNodeKind("Blockquote") + +// Kind implements Node.Kind. +func (n *Blockquote) Kind() NodeKind { + return KindBlockquote +} + +// NewBlockquote returns a new Blockquote node. +func NewBlockquote() *Blockquote { + return &Blockquote{ + BaseBlock: BaseBlock{}, + } +} + +// A List struct represents a list of Markdown text. +type List struct { + BaseBlock + + // Marker is a marker character like '-', '+', ')' and '.'. + Marker byte + + // IsTight is a true if this list is a 'tight' list. + // See https://spec.commonmark.org/0.30/#loose for details. + IsTight bool + + // Start is an initial number of this ordered list. + // If this list is not an ordered list, Start is 0. + Start int +} + +// IsOrdered returns true if this list is an ordered list, otherwise false. +func (l *List) IsOrdered() bool { + return l.Marker == '.' || l.Marker == ')' +} + +// CanContinue returns true if this list can continue with +// the given mark and a list type, otherwise false. +func (l *List) CanContinue(marker byte, isOrdered bool) bool { + return marker == l.Marker && isOrdered == l.IsOrdered() +} + +// Dump implements Node.Dump. +func (l *List) Dump(source []byte, level int) { + m := map[string]string{ + "Ordered": fmt.Sprintf("%v", l.IsOrdered()), + "Marker": fmt.Sprintf("%c", l.Marker), + "Tight": fmt.Sprintf("%v", l.IsTight), + } + if l.IsOrdered() { + m["Start"] = fmt.Sprintf("%d", l.Start) + } + DumpHelper(l, source, level, m, nil) +} + +// KindList is a NodeKind of the List node. +var KindList = NewNodeKind("List") + +// Kind implements Node.Kind. +func (l *List) Kind() NodeKind { + return KindList +} + +// NewList returns a new List node. +func NewList(marker byte) *List { + return &List{ + BaseBlock: BaseBlock{}, + Marker: marker, + IsTight: true, + } +} + +// A ListItem struct represents a list item of Markdown text. +type ListItem struct { + BaseBlock + + // Offset is an offset position of this item. + Offset int +} + +// Dump implements Node.Dump. +func (n *ListItem) Dump(source []byte, level int) { + m := map[string]string{ + "Offset": fmt.Sprintf("%d", n.Offset), + } + DumpHelper(n, source, level, m, nil) +} + +// KindListItem is a NodeKind of the ListItem node. +var KindListItem = NewNodeKind("ListItem") + +// Kind implements Node.Kind. +func (n *ListItem) Kind() NodeKind { + return KindListItem +} + +// NewListItem returns a new ListItem node. +func NewListItem(offset int) *ListItem { + return &ListItem{ + BaseBlock: BaseBlock{}, + Offset: offset, + } +} + +// HTMLBlockType represents kinds of an html blocks. +// See https://spec.commonmark.org/0.30/#html-blocks +type HTMLBlockType int + +const ( + // HTMLBlockType1 represents type 1 html blocks + HTMLBlockType1 HTMLBlockType = iota + 1 + // HTMLBlockType2 represents type 2 html blocks + HTMLBlockType2 + // HTMLBlockType3 represents type 3 html blocks + HTMLBlockType3 + // HTMLBlockType4 represents type 4 html blocks + HTMLBlockType4 + // HTMLBlockType5 represents type 5 html blocks + HTMLBlockType5 + // HTMLBlockType6 represents type 6 html blocks + HTMLBlockType6 + // HTMLBlockType7 represents type 7 html blocks + HTMLBlockType7 +) + +// An HTMLBlock struct represents an html block of Markdown text. +type HTMLBlock struct { + BaseBlock + + // Type is a type of this html block. + HTMLBlockType HTMLBlockType + + // ClosureLine is a line that closes this html block. + ClosureLine textm.Segment +} + +// IsRaw implements Node.IsRaw. +func (n *HTMLBlock) IsRaw() bool { + return true +} + +// HasClosure returns true if this html block has a closure line, +// otherwise false. +func (n *HTMLBlock) HasClosure() bool { + return n.ClosureLine.Start >= 0 +} + +// Dump implements Node.Dump. +func (n *HTMLBlock) Dump(source []byte, level int) { + indent := strings.Repeat(" ", level) + fmt.Printf("%s%s {\n", indent, "HTMLBlock") + indent2 := strings.Repeat(" ", level+1) + fmt.Printf("%sRawText: \"", indent2) + for i := 0; i < n.Lines().Len(); i++ { + s := n.Lines().At(i) + fmt.Print(string(source[s.Start:s.Stop])) + } + fmt.Printf("\"\n") + for c := n.FirstChild(); c != nil; c = c.NextSibling() { + c.Dump(source, level+1) + } + if n.HasClosure() { + cl := n.ClosureLine + fmt.Printf("%sClosure: \"%s\"\n", indent2, string(cl.Value(source))) + } + fmt.Printf("%s}\n", indent) +} + +// KindHTMLBlock is a NodeKind of the HTMLBlock node. +var KindHTMLBlock = NewNodeKind("HTMLBlock") + +// Kind implements Node.Kind. +func (n *HTMLBlock) Kind() NodeKind { + return KindHTMLBlock +} + +// NewHTMLBlock returns a new HTMLBlock node. +func NewHTMLBlock(typ HTMLBlockType) *HTMLBlock { + return &HTMLBlock{ + BaseBlock: BaseBlock{}, + HTMLBlockType: typ, + ClosureLine: textm.NewSegment(-1, -1), + } +} diff --git a/vendor/github.com/yuin/goldmark/ast/inline.go b/vendor/github.com/yuin/goldmark/ast/inline.go new file mode 100644 index 00000000..fa6fc34f --- /dev/null +++ b/vendor/github.com/yuin/goldmark/ast/inline.go @@ -0,0 +1,548 @@ +package ast + +import ( + "fmt" + "strings" + + textm "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +// A BaseInline struct implements the Node interface partialliy. +type BaseInline struct { + BaseNode +} + +// Type implements Node.Type +func (b *BaseInline) Type() NodeType { + return TypeInline +} + +// IsRaw implements Node.IsRaw +func (b *BaseInline) IsRaw() bool { + return false +} + +// HasBlankPreviousLines implements Node.HasBlankPreviousLines. +func (b *BaseInline) HasBlankPreviousLines() bool { + panic("can not call with inline nodes.") +} + +// SetBlankPreviousLines implements Node.SetBlankPreviousLines. +func (b *BaseInline) SetBlankPreviousLines(v bool) { + panic("can not call with inline nodes.") +} + +// Lines implements Node.Lines +func (b *BaseInline) Lines() *textm.Segments { + panic("can not call with inline nodes.") +} + +// SetLines implements Node.SetLines +func (b *BaseInline) SetLines(v *textm.Segments) { + panic("can not call with inline nodes.") +} + +// A Text struct represents a textual content of the Markdown text. +type Text struct { + BaseInline + // Segment is a position in a source text. + Segment textm.Segment + + flags uint8 +} + +const ( + textSoftLineBreak = 1 << iota + textHardLineBreak + textRaw + textCode +) + +func textFlagsString(flags uint8) string { + buf := []string{} + if flags&textSoftLineBreak != 0 { + buf = append(buf, "SoftLineBreak") + } + if flags&textHardLineBreak != 0 { + buf = append(buf, "HardLineBreak") + } + if flags&textRaw != 0 { + buf = append(buf, "Raw") + } + if flags&textCode != 0 { + buf = append(buf, "Code") + } + return strings.Join(buf, ", ") +} + +// Inline implements Inline.Inline. +func (n *Text) Inline() { +} + +// SoftLineBreak returns true if this node ends with a new line, +// otherwise false. +func (n *Text) SoftLineBreak() bool { + return n.flags&textSoftLineBreak != 0 +} + +// SetSoftLineBreak sets whether this node ends with a new line. +func (n *Text) SetSoftLineBreak(v bool) { + if v { + n.flags |= textSoftLineBreak + } else { + n.flags = n.flags &^ textHardLineBreak + } +} + +// IsRaw returns true if this text should be rendered without unescaping +// back slash escapes and resolving references. +func (n *Text) IsRaw() bool { + return n.flags&textRaw != 0 +} + +// SetRaw sets whether this text should be rendered as raw contents. +func (n *Text) SetRaw(v bool) { + if v { + n.flags |= textRaw + } else { + n.flags = n.flags &^ textRaw + } +} + +// HardLineBreak returns true if this node ends with a hard line break. +// See https://spec.commonmark.org/0.30/#hard-line-breaks for details. +func (n *Text) HardLineBreak() bool { + return n.flags&textHardLineBreak != 0 +} + +// SetHardLineBreak sets whether this node ends with a hard line break. +func (n *Text) SetHardLineBreak(v bool) { + if v { + n.flags |= textHardLineBreak + } else { + n.flags = n.flags &^ textHardLineBreak + } +} + +// Merge merges a Node n into this node. +// Merge returns true if the given node has been merged, otherwise false. +func (n *Text) Merge(node Node, source []byte) bool { + t, ok := node.(*Text) + if !ok { + return false + } + if n.Segment.Stop != t.Segment.Start || t.Segment.Padding != 0 || source[n.Segment.Stop-1] == '\n' || t.IsRaw() != n.IsRaw() { + return false + } + n.Segment.Stop = t.Segment.Stop + n.SetSoftLineBreak(t.SoftLineBreak()) + n.SetHardLineBreak(t.HardLineBreak()) + return true +} + +// Text implements Node.Text. +func (n *Text) Text(source []byte) []byte { + return n.Segment.Value(source) +} + +// Dump implements Node.Dump. +func (n *Text) Dump(source []byte, level int) { + fs := textFlagsString(n.flags) + if len(fs) != 0 { + fs = "(" + fs + ")" + } + fmt.Printf("%sText%s: \"%s\"\n", strings.Repeat(" ", level), fs, strings.TrimRight(string(n.Text(source)), "\n")) +} + +// KindText is a NodeKind of the Text node. +var KindText = NewNodeKind("Text") + +// Kind implements Node.Kind. +func (n *Text) Kind() NodeKind { + return KindText +} + +// NewText returns a new Text node. +func NewText() *Text { + return &Text{ + BaseInline: BaseInline{}, + } +} + +// NewTextSegment returns a new Text node with the given source position. +func NewTextSegment(v textm.Segment) *Text { + return &Text{ + BaseInline: BaseInline{}, + Segment: v, + } +} + +// NewRawTextSegment returns a new Text node with the given source position. +// The new node should be rendered as raw contents. +func NewRawTextSegment(v textm.Segment) *Text { + t := &Text{ + BaseInline: BaseInline{}, + Segment: v, + } + t.SetRaw(true) + return t +} + +// MergeOrAppendTextSegment merges a given s into the last child of the parent if +// it can be merged, otherwise creates a new Text node and appends it to after current +// last child. +func MergeOrAppendTextSegment(parent Node, s textm.Segment) { + last := parent.LastChild() + t, ok := last.(*Text) + if ok && t.Segment.Stop == s.Start && !t.SoftLineBreak() { + t.Segment = t.Segment.WithStop(s.Stop) + } else { + parent.AppendChild(parent, NewTextSegment(s)) + } +} + +// MergeOrReplaceTextSegment merges a given s into a previous sibling of the node n +// if a previous sibling of the node n is *Text, otherwise replaces Node n with s. +func MergeOrReplaceTextSegment(parent Node, n Node, s textm.Segment) { + prev := n.PreviousSibling() + if t, ok := prev.(*Text); ok && t.Segment.Stop == s.Start && !t.SoftLineBreak() { + t.Segment = t.Segment.WithStop(s.Stop) + parent.RemoveChild(parent, n) + } else { + parent.ReplaceChild(parent, n, NewTextSegment(s)) + } +} + +// A String struct is a textual content that has a concrete value +type String struct { + BaseInline + + Value []byte + flags uint8 +} + +// Inline implements Inline.Inline. +func (n *String) Inline() { +} + +// IsRaw returns true if this text should be rendered without unescaping +// back slash escapes and resolving references. +func (n *String) IsRaw() bool { + return n.flags&textRaw != 0 +} + +// SetRaw sets whether this text should be rendered as raw contents. +func (n *String) SetRaw(v bool) { + if v { + n.flags |= textRaw + } else { + n.flags = n.flags &^ textRaw + } +} + +// IsCode returns true if this text should be rendered without any +// modifications. +func (n *String) IsCode() bool { + return n.flags&textCode != 0 +} + +// SetCode sets whether this text should be rendered without any modifications. +func (n *String) SetCode(v bool) { + if v { + n.flags |= textCode + } else { + n.flags = n.flags &^ textCode + } +} + +// Text implements Node.Text. +func (n *String) Text(source []byte) []byte { + return n.Value +} + +// Dump implements Node.Dump. +func (n *String) Dump(source []byte, level int) { + fs := textFlagsString(n.flags) + if len(fs) != 0 { + fs = "(" + fs + ")" + } + fmt.Printf("%sString%s: \"%s\"\n", strings.Repeat(" ", level), fs, strings.TrimRight(string(n.Value), "\n")) +} + +// KindString is a NodeKind of the String node. +var KindString = NewNodeKind("String") + +// Kind implements Node.Kind. +func (n *String) Kind() NodeKind { + return KindString +} + +// NewString returns a new String node. +func NewString(v []byte) *String { + return &String{ + Value: v, + } +} + +// A CodeSpan struct represents a code span of Markdown text. +type CodeSpan struct { + BaseInline +} + +// Inline implements Inline.Inline . +func (n *CodeSpan) Inline() { +} + +// IsBlank returns true if this node consists of spaces, otherwise false. +func (n *CodeSpan) IsBlank(source []byte) bool { + for c := n.FirstChild(); c != nil; c = c.NextSibling() { + text := c.(*Text).Segment + if !util.IsBlank(text.Value(source)) { + return false + } + } + return true +} + +// Dump implements Node.Dump +func (n *CodeSpan) Dump(source []byte, level int) { + DumpHelper(n, source, level, nil, nil) +} + +// KindCodeSpan is a NodeKind of the CodeSpan node. +var KindCodeSpan = NewNodeKind("CodeSpan") + +// Kind implements Node.Kind. +func (n *CodeSpan) Kind() NodeKind { + return KindCodeSpan +} + +// NewCodeSpan returns a new CodeSpan node. +func NewCodeSpan() *CodeSpan { + return &CodeSpan{ + BaseInline: BaseInline{}, + } +} + +// An Emphasis struct represents an emphasis of Markdown text. +type Emphasis struct { + BaseInline + + // Level is a level of the emphasis. + Level int +} + +// Dump implements Node.Dump. +func (n *Emphasis) Dump(source []byte, level int) { + m := map[string]string{ + "Level": fmt.Sprintf("%v", n.Level), + } + DumpHelper(n, source, level, m, nil) +} + +// KindEmphasis is a NodeKind of the Emphasis node. +var KindEmphasis = NewNodeKind("Emphasis") + +// Kind implements Node.Kind. +func (n *Emphasis) Kind() NodeKind { + return KindEmphasis +} + +// NewEmphasis returns a new Emphasis node with the given level. +func NewEmphasis(level int) *Emphasis { + return &Emphasis{ + BaseInline: BaseInline{}, + Level: level, + } +} + +type baseLink struct { + BaseInline + + // Destination is a destination(URL) of this link. + Destination []byte + + // Title is a title of this link. + Title []byte +} + +// Inline implements Inline.Inline. +func (n *baseLink) Inline() { +} + +// A Link struct represents a link of the Markdown text. +type Link struct { + baseLink +} + +// Dump implements Node.Dump. +func (n *Link) Dump(source []byte, level int) { + m := map[string]string{} + m["Destination"] = string(n.Destination) + m["Title"] = string(n.Title) + DumpHelper(n, source, level, m, nil) +} + +// KindLink is a NodeKind of the Link node. +var KindLink = NewNodeKind("Link") + +// Kind implements Node.Kind. +func (n *Link) Kind() NodeKind { + return KindLink +} + +// NewLink returns a new Link node. +func NewLink() *Link { + c := &Link{ + baseLink: baseLink{ + BaseInline: BaseInline{}, + }, + } + return c +} + +// An Image struct represents an image of the Markdown text. +type Image struct { + baseLink +} + +// Dump implements Node.Dump. +func (n *Image) Dump(source []byte, level int) { + m := map[string]string{} + m["Destination"] = string(n.Destination) + m["Title"] = string(n.Title) + DumpHelper(n, source, level, m, nil) +} + +// KindImage is a NodeKind of the Image node. +var KindImage = NewNodeKind("Image") + +// Kind implements Node.Kind. +func (n *Image) Kind() NodeKind { + return KindImage +} + +// NewImage returns a new Image node. +func NewImage(link *Link) *Image { + c := &Image{ + baseLink: baseLink{ + BaseInline: BaseInline{}, + }, + } + c.Destination = link.Destination + c.Title = link.Title + for n := link.FirstChild(); n != nil; { + next := n.NextSibling() + link.RemoveChild(link, n) + c.AppendChild(c, n) + n = next + } + + return c +} + +// AutoLinkType defines kind of auto links. +type AutoLinkType int + +const ( + // AutoLinkEmail indicates that an autolink is an email address. + AutoLinkEmail AutoLinkType = iota + 1 + // AutoLinkURL indicates that an autolink is a generic URL. + AutoLinkURL +) + +// An AutoLink struct represents an autolink of the Markdown text. +type AutoLink struct { + BaseInline + // Type is a type of this autolink. + AutoLinkType AutoLinkType + + // Protocol specified a protocol of the link. + Protocol []byte + + value *Text +} + +// Inline implements Inline.Inline. +func (n *AutoLink) Inline() {} + +// Dump implements Node.Dump +func (n *AutoLink) Dump(source []byte, level int) { + segment := n.value.Segment + m := map[string]string{ + "Value": string(segment.Value(source)), + } + DumpHelper(n, source, level, m, nil) +} + +// KindAutoLink is a NodeKind of the AutoLink node. +var KindAutoLink = NewNodeKind("AutoLink") + +// Kind implements Node.Kind. +func (n *AutoLink) Kind() NodeKind { + return KindAutoLink +} + +// URL returns an url of this node. +func (n *AutoLink) URL(source []byte) []byte { + if n.Protocol != nil { + s := n.value.Segment + ret := make([]byte, 0, len(n.Protocol)+s.Len()+3) + ret = append(ret, n.Protocol...) + ret = append(ret, ':', '/', '/') + ret = append(ret, n.value.Text(source)...) + return ret + } + return n.value.Text(source) +} + +// Label returns a label of this node. +func (n *AutoLink) Label(source []byte) []byte { + return n.value.Text(source) +} + +// NewAutoLink returns a new AutoLink node. +func NewAutoLink(typ AutoLinkType, value *Text) *AutoLink { + return &AutoLink{ + BaseInline: BaseInline{}, + value: value, + AutoLinkType: typ, + } +} + +// A RawHTML struct represents an inline raw HTML of the Markdown text. +type RawHTML struct { + BaseInline + Segments *textm.Segments +} + +// Inline implements Inline.Inline. +func (n *RawHTML) Inline() {} + +// Dump implements Node.Dump. +func (n *RawHTML) Dump(source []byte, level int) { + m := map[string]string{} + t := []string{} + for i := 0; i < n.Segments.Len(); i++ { + segment := n.Segments.At(i) + t = append(t, string(segment.Value(source))) + } + m["RawText"] = strings.Join(t, "") + DumpHelper(n, source, level, m, nil) +} + +// KindRawHTML is a NodeKind of the RawHTML node. +var KindRawHTML = NewNodeKind("RawHTML") + +// Kind implements Node.Kind. +func (n *RawHTML) Kind() NodeKind { + return KindRawHTML +} + +// NewRawHTML returns a new RawHTML node. +func NewRawHTML() *RawHTML { + return &RawHTML{ + Segments: textm.NewSegments(), + } +} diff --git a/vendor/github.com/yuin/goldmark/extension/ast/definition_list.go b/vendor/github.com/yuin/goldmark/extension/ast/definition_list.go new file mode 100644 index 00000000..1beffb3a --- /dev/null +++ b/vendor/github.com/yuin/goldmark/extension/ast/definition_list.go @@ -0,0 +1,83 @@ +package ast + +import ( + gast "github.com/yuin/goldmark/ast" +) + +// A DefinitionList struct represents a definition list of Markdown +// (PHPMarkdownExtra) text. +type DefinitionList struct { + gast.BaseBlock + Offset int + TemporaryParagraph *gast.Paragraph +} + +// Dump implements Node.Dump. +func (n *DefinitionList) Dump(source []byte, level int) { + gast.DumpHelper(n, source, level, nil, nil) +} + +// KindDefinitionList is a NodeKind of the DefinitionList node. +var KindDefinitionList = gast.NewNodeKind("DefinitionList") + +// Kind implements Node.Kind. +func (n *DefinitionList) Kind() gast.NodeKind { + return KindDefinitionList +} + +// NewDefinitionList returns a new DefinitionList node. +func NewDefinitionList(offset int, para *gast.Paragraph) *DefinitionList { + return &DefinitionList{ + Offset: offset, + TemporaryParagraph: para, + } +} + +// A DefinitionTerm struct represents a definition list term of Markdown +// (PHPMarkdownExtra) text. +type DefinitionTerm struct { + gast.BaseBlock +} + +// Dump implements Node.Dump. +func (n *DefinitionTerm) Dump(source []byte, level int) { + gast.DumpHelper(n, source, level, nil, nil) +} + +// KindDefinitionTerm is a NodeKind of the DefinitionTerm node. +var KindDefinitionTerm = gast.NewNodeKind("DefinitionTerm") + +// Kind implements Node.Kind. +func (n *DefinitionTerm) Kind() gast.NodeKind { + return KindDefinitionTerm +} + +// NewDefinitionTerm returns a new DefinitionTerm node. +func NewDefinitionTerm() *DefinitionTerm { + return &DefinitionTerm{} +} + +// A DefinitionDescription struct represents a definition list description of Markdown +// (PHPMarkdownExtra) text. +type DefinitionDescription struct { + gast.BaseBlock + IsTight bool +} + +// Dump implements Node.Dump. +func (n *DefinitionDescription) Dump(source []byte, level int) { + gast.DumpHelper(n, source, level, nil, nil) +} + +// KindDefinitionDescription is a NodeKind of the DefinitionDescription node. +var KindDefinitionDescription = gast.NewNodeKind("DefinitionDescription") + +// Kind implements Node.Kind. +func (n *DefinitionDescription) Kind() gast.NodeKind { + return KindDefinitionDescription +} + +// NewDefinitionDescription returns a new DefinitionDescription node. +func NewDefinitionDescription() *DefinitionDescription { + return &DefinitionDescription{} +} diff --git a/vendor/github.com/yuin/goldmark/extension/ast/footnote.go b/vendor/github.com/yuin/goldmark/extension/ast/footnote.go new file mode 100644 index 00000000..dedbab4f --- /dev/null +++ b/vendor/github.com/yuin/goldmark/extension/ast/footnote.go @@ -0,0 +1,132 @@ +package ast + +import ( + "fmt" + + gast "github.com/yuin/goldmark/ast" +) + +// A FootnoteLink struct represents a link to a footnote of Markdown +// (PHP Markdown Extra) text. +type FootnoteLink struct { + gast.BaseInline + Index int + RefCount int +} + +// Dump implements Node.Dump. +func (n *FootnoteLink) Dump(source []byte, level int) { + m := map[string]string{} + m["Index"] = fmt.Sprintf("%v", n.Index) + m["RefCount"] = fmt.Sprintf("%v", n.RefCount) + gast.DumpHelper(n, source, level, m, nil) +} + +// KindFootnoteLink is a NodeKind of the FootnoteLink node. +var KindFootnoteLink = gast.NewNodeKind("FootnoteLink") + +// Kind implements Node.Kind. +func (n *FootnoteLink) Kind() gast.NodeKind { + return KindFootnoteLink +} + +// NewFootnoteLink returns a new FootnoteLink node. +func NewFootnoteLink(index int) *FootnoteLink { + return &FootnoteLink{ + Index: index, + RefCount: 0, + } +} + +// A FootnoteBacklink struct represents a link to a footnote of Markdown +// (PHP Markdown Extra) text. +type FootnoteBacklink struct { + gast.BaseInline + Index int + RefCount int +} + +// Dump implements Node.Dump. +func (n *FootnoteBacklink) Dump(source []byte, level int) { + m := map[string]string{} + m["Index"] = fmt.Sprintf("%v", n.Index) + m["RefCount"] = fmt.Sprintf("%v", n.RefCount) + gast.DumpHelper(n, source, level, m, nil) +} + +// KindFootnoteBacklink is a NodeKind of the FootnoteBacklink node. +var KindFootnoteBacklink = gast.NewNodeKind("FootnoteBacklink") + +// Kind implements Node.Kind. +func (n *FootnoteBacklink) Kind() gast.NodeKind { + return KindFootnoteBacklink +} + +// NewFootnoteBacklink returns a new FootnoteBacklink node. +func NewFootnoteBacklink(index int) *FootnoteBacklink { + return &FootnoteBacklink{ + Index: index, + RefCount: 0, + } +} + +// A Footnote struct represents a footnote of Markdown +// (PHP Markdown Extra) text. +type Footnote struct { + gast.BaseBlock + Ref []byte + Index int +} + +// Dump implements Node.Dump. +func (n *Footnote) Dump(source []byte, level int) { + m := map[string]string{} + m["Index"] = fmt.Sprintf("%v", n.Index) + m["Ref"] = fmt.Sprintf("%s", n.Ref) + gast.DumpHelper(n, source, level, m, nil) +} + +// KindFootnote is a NodeKind of the Footnote node. +var KindFootnote = gast.NewNodeKind("Footnote") + +// Kind implements Node.Kind. +func (n *Footnote) Kind() gast.NodeKind { + return KindFootnote +} + +// NewFootnote returns a new Footnote node. +func NewFootnote(ref []byte) *Footnote { + return &Footnote{ + Ref: ref, + Index: -1, + } +} + +// A FootnoteList struct represents footnotes of Markdown +// (PHP Markdown Extra) text. +type FootnoteList struct { + gast.BaseBlock + Count int +} + +// Dump implements Node.Dump. +func (n *FootnoteList) Dump(source []byte, level int) { + m := map[string]string{} + m["Count"] = fmt.Sprintf("%v", n.Count) + gast.DumpHelper(n, source, level, m, nil) +} + +// KindFootnoteList is a NodeKind of the FootnoteList node. +var KindFootnoteList = gast.NewNodeKind("FootnoteList") + +// Kind implements Node.Kind. +func (n *FootnoteList) Kind() gast.NodeKind { + return KindFootnoteList +} + +// NewFootnoteList returns a new FootnoteList node. +func NewFootnoteList() *FootnoteList { + return &FootnoteList{ + Count: 0, + } +} diff --git a/vendor/github.com/yuin/goldmark/extension/ast/strikethrough.go b/vendor/github.com/yuin/goldmark/extension/ast/strikethrough.go new file mode 100644 index 00000000..a9216b72 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/extension/ast/strikethrough.go @@ -0,0 +1,29 @@ +// Package ast defines AST nodes that represents extension's elements +package ast + +import ( + gast "github.com/yuin/goldmark/ast" +) + +// A Strikethrough struct represents a strikethrough of GFM text. +type Strikethrough struct { + gast.BaseInline +} + +// Dump implements Node.Dump. +func (n *Strikethrough) Dump(source []byte, level int) { + gast.DumpHelper(n, source, level, nil, nil) +} + +// KindStrikethrough is a NodeKind of the Strikethrough node. +var KindStrikethrough = gast.NewNodeKind("Strikethrough") + +// Kind implements Node.Kind. +func (n *Strikethrough) Kind() gast.NodeKind { + return KindStrikethrough +} + +// NewStrikethrough returns a new Strikethrough node. +func NewStrikethrough() *Strikethrough { + return &Strikethrough{} +} diff --git a/vendor/github.com/yuin/goldmark/extension/ast/table.go b/vendor/github.com/yuin/goldmark/extension/ast/table.go new file mode 100644 index 00000000..1d8890b5 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/extension/ast/table.go @@ -0,0 +1,157 @@ +package ast + +import ( + "fmt" + gast "github.com/yuin/goldmark/ast" + "strings" +) + +// Alignment is a text alignment of table cells. +type Alignment int + +const ( + // AlignLeft indicates text should be left justified. + AlignLeft Alignment = iota + 1 + + // AlignRight indicates text should be right justified. + AlignRight + + // AlignCenter indicates text should be centered. + AlignCenter + + // AlignNone indicates text should be aligned by default manner. + AlignNone +) + +func (a Alignment) String() string { + switch a { + case AlignLeft: + return "left" + case AlignRight: + return "right" + case AlignCenter: + return "center" + case AlignNone: + return "none" + } + return "" +} + +// A Table struct represents a table of Markdown(GFM) text. +type Table struct { + gast.BaseBlock + + // Alignments returns alignments of the columns. + Alignments []Alignment +} + +// Dump implements Node.Dump +func (n *Table) Dump(source []byte, level int) { + gast.DumpHelper(n, source, level, nil, func(level int) { + indent := strings.Repeat(" ", level) + fmt.Printf("%sAlignments {\n", indent) + for i, alignment := range n.Alignments { + indent2 := strings.Repeat(" ", level+1) + fmt.Printf("%s%s", indent2, alignment.String()) + if i != len(n.Alignments)-1 { + fmt.Println("") + } + } + fmt.Printf("\n%s}\n", indent) + }) +} + +// KindTable is a NodeKind of the Table node. +var KindTable = gast.NewNodeKind("Table") + +// Kind implements Node.Kind. +func (n *Table) Kind() gast.NodeKind { + return KindTable +} + +// NewTable returns a new Table node. +func NewTable() *Table { + return &Table{ + Alignments: []Alignment{}, + } +} + +// A TableRow struct represents a table row of Markdown(GFM) text. +type TableRow struct { + gast.BaseBlock + Alignments []Alignment +} + +// Dump implements Node.Dump. +func (n *TableRow) Dump(source []byte, level int) { + gast.DumpHelper(n, source, level, nil, nil) +} + +// KindTableRow is a NodeKind of the TableRow node. +var KindTableRow = gast.NewNodeKind("TableRow") + +// Kind implements Node.Kind. +func (n *TableRow) Kind() gast.NodeKind { + return KindTableRow +} + +// NewTableRow returns a new TableRow node. +func NewTableRow(alignments []Alignment) *TableRow { + return &TableRow{} +} + +// A TableHeader struct represents a table header of Markdown(GFM) text. +type TableHeader struct { + gast.BaseBlock + Alignments []Alignment +} + +// KindTableHeader is a NodeKind of the TableHeader node. +var KindTableHeader = gast.NewNodeKind("TableHeader") + +// Kind implements Node.Kind. +func (n *TableHeader) Kind() gast.NodeKind { + return KindTableHeader +} + +// Dump implements Node.Dump. +func (n *TableHeader) Dump(source []byte, level int) { + gast.DumpHelper(n, source, level, nil, nil) +} + +// NewTableHeader returns a new TableHeader node. +func NewTableHeader(row *TableRow) *TableHeader { + n := &TableHeader{} + for c := row.FirstChild(); c != nil; { + next := c.NextSibling() + n.AppendChild(n, c) + c = next + } + return n +} + +// A TableCell struct represents a table cell of a Markdown(GFM) text. +type TableCell struct { + gast.BaseBlock + Alignment Alignment +} + +// Dump implements Node.Dump. +func (n *TableCell) Dump(source []byte, level int) { + gast.DumpHelper(n, source, level, nil, nil) +} + +// KindTableCell is a NodeKind of the TableCell node. +var KindTableCell = gast.NewNodeKind("TableCell") + +// Kind implements Node.Kind. +func (n *TableCell) Kind() gast.NodeKind { + return KindTableCell +} + +// NewTableCell returns a new TableCell node. +func NewTableCell() *TableCell { + return &TableCell{ + Alignment: AlignNone, + } +} diff --git a/vendor/github.com/yuin/goldmark/extension/ast/tasklist.go b/vendor/github.com/yuin/goldmark/extension/ast/tasklist.go new file mode 100644 index 00000000..670cc149 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/extension/ast/tasklist.go @@ -0,0 +1,35 @@ +package ast + +import ( + "fmt" + gast "github.com/yuin/goldmark/ast" +) + +// A TaskCheckBox struct represents a checkbox of a task list. +type TaskCheckBox struct { + gast.BaseInline + IsChecked bool +} + +// Dump implements Node.Dump. +func (n *TaskCheckBox) Dump(source []byte, level int) { + m := map[string]string{ + "Checked": fmt.Sprintf("%v", n.IsChecked), + } + gast.DumpHelper(n, source, level, m, nil) +} + +// KindTaskCheckBox is a NodeKind of the TaskCheckBox node. +var KindTaskCheckBox = gast.NewNodeKind("TaskCheckBox") + +// Kind implements Node.Kind. +func (n *TaskCheckBox) Kind() gast.NodeKind { + return KindTaskCheckBox +} + +// NewTaskCheckBox returns a new TaskCheckBox node. +func NewTaskCheckBox(checked bool) *TaskCheckBox { + return &TaskCheckBox{ + IsChecked: checked, + } +} diff --git a/vendor/github.com/yuin/goldmark/extension/definition_list.go b/vendor/github.com/yuin/goldmark/extension/definition_list.go new file mode 100644 index 00000000..d2f5fecb --- /dev/null +++ b/vendor/github.com/yuin/goldmark/extension/definition_list.go @@ -0,0 +1,270 @@ +package extension + +import ( + "github.com/yuin/goldmark" + gast "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +type definitionListParser struct { +} + +var defaultDefinitionListParser = &definitionListParser{} + +// NewDefinitionListParser return a new parser.BlockParser that +// can parse PHP Markdown Extra Definition lists. +func NewDefinitionListParser() parser.BlockParser { + return defaultDefinitionListParser +} + +func (b *definitionListParser) Trigger() []byte { + return []byte{':'} +} + +func (b *definitionListParser) Open(parent gast.Node, reader text.Reader, pc parser.Context) (gast.Node, parser.State) { + if _, ok := parent.(*ast.DefinitionList); ok { + return nil, parser.NoChildren + } + line, _ := reader.PeekLine() + pos := pc.BlockOffset() + indent := pc.BlockIndent() + if pos < 0 || line[pos] != ':' || indent != 0 { + return nil, parser.NoChildren + } + + last := parent.LastChild() + // need 1 or more spaces after ':' + w, _ := util.IndentWidth(line[pos+1:], pos+1) + if w < 1 { + return nil, parser.NoChildren + } + if w >= 8 { // starts with indented code + w = 5 + } + w += pos + 1 /* 1 = ':' */ + + para, lastIsParagraph := last.(*gast.Paragraph) + var list *ast.DefinitionList + status := parser.HasChildren + var ok bool + if lastIsParagraph { + list, ok = last.PreviousSibling().(*ast.DefinitionList) + if ok { // is not first item + list.Offset = w + list.TemporaryParagraph = para + } else { // is first item + list = ast.NewDefinitionList(w, para) + status |= parser.RequireParagraph + } + } else if list, ok = last.(*ast.DefinitionList); ok { // multiple description + list.Offset = w + list.TemporaryParagraph = nil + } else { + return nil, parser.NoChildren + } + + return list, status +} + +func (b *definitionListParser) Continue(node gast.Node, reader text.Reader, pc parser.Context) parser.State { + line, _ := reader.PeekLine() + if util.IsBlank(line) { + return parser.Continue | parser.HasChildren + } + list, _ := node.(*ast.DefinitionList) + w, _ := util.IndentWidth(line, reader.LineOffset()) + if w < list.Offset { + return parser.Close + } + pos, padding := util.IndentPosition(line, reader.LineOffset(), list.Offset) + reader.AdvanceAndSetPadding(pos, padding) + return parser.Continue | parser.HasChildren +} + +func (b *definitionListParser) Close(node gast.Node, reader text.Reader, pc parser.Context) { + // nothing to do +} + +func (b *definitionListParser) CanInterruptParagraph() bool { + return true +} + +func (b *definitionListParser) CanAcceptIndentedLine() bool { + return false +} + +type definitionDescriptionParser struct { +} + +var defaultDefinitionDescriptionParser = &definitionDescriptionParser{} + +// NewDefinitionDescriptionParser return a new parser.BlockParser that +// can parse definition description starts with ':'. +func NewDefinitionDescriptionParser() parser.BlockParser { + return defaultDefinitionDescriptionParser +} + +func (b *definitionDescriptionParser) Trigger() []byte { + return []byte{':'} +} + +func (b *definitionDescriptionParser) Open(parent gast.Node, reader text.Reader, pc parser.Context) (gast.Node, parser.State) { + line, _ := reader.PeekLine() + pos := pc.BlockOffset() + indent := pc.BlockIndent() + if pos < 0 || line[pos] != ':' || indent != 0 { + return nil, parser.NoChildren + } + list, _ := parent.(*ast.DefinitionList) + if list == nil { + return nil, parser.NoChildren + } + para := list.TemporaryParagraph + list.TemporaryParagraph = nil + if para != nil { + lines := para.Lines() + l := lines.Len() + for i := 0; i < l; i++ { + term := ast.NewDefinitionTerm() + segment := lines.At(i) + term.Lines().Append(segment.TrimRightSpace(reader.Source())) + list.AppendChild(list, term) + } + para.Parent().RemoveChild(para.Parent(), para) + } + cpos, padding := util.IndentPosition(line[pos+1:], pos+1, list.Offset-pos-1) + reader.AdvanceAndSetPadding(cpos+1, padding) + + return ast.NewDefinitionDescription(), parser.HasChildren +} + +func (b *definitionDescriptionParser) Continue(node gast.Node, reader text.Reader, pc parser.Context) parser.State { + // definitionListParser detects end of the description. + // so this method will never be called. + return parser.Continue | parser.HasChildren +} + +func (b *definitionDescriptionParser) Close(node gast.Node, reader text.Reader, pc parser.Context) { + desc := node.(*ast.DefinitionDescription) + desc.IsTight = !desc.HasBlankPreviousLines() + if desc.IsTight { + for gc := desc.FirstChild(); gc != nil; gc = gc.NextSibling() { + paragraph, ok := gc.(*gast.Paragraph) + if ok { + textBlock := gast.NewTextBlock() + textBlock.SetLines(paragraph.Lines()) + desc.ReplaceChild(desc, paragraph, textBlock) + } + } + } +} + +func (b *definitionDescriptionParser) CanInterruptParagraph() bool { + return true +} + +func (b *definitionDescriptionParser) CanAcceptIndentedLine() bool { + return false +} + +// DefinitionListHTMLRenderer is a renderer.NodeRenderer implementation that +// renders DefinitionList nodes. +type DefinitionListHTMLRenderer struct { + html.Config +} + +// NewDefinitionListHTMLRenderer returns a new DefinitionListHTMLRenderer. +func NewDefinitionListHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { + r := &DefinitionListHTMLRenderer{ + Config: html.NewConfig(), + } + for _, opt := range opts { + opt.SetHTMLOption(&r.Config) + } + return r +} + +// RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs. +func (r *DefinitionListHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(ast.KindDefinitionList, r.renderDefinitionList) + reg.Register(ast.KindDefinitionTerm, r.renderDefinitionTerm) + reg.Register(ast.KindDefinitionDescription, r.renderDefinitionDescription) +} + +// DefinitionListAttributeFilter defines attribute names which dl elements can have. +var DefinitionListAttributeFilter = html.GlobalAttributeFilter + +func (r *DefinitionListHTMLRenderer) renderDefinitionList(w util.BufWriter, source []byte, n gast.Node, entering bool) (gast.WalkStatus, error) { + if entering { + if n.Attributes() != nil { + _, _ = w.WriteString("<dl") + html.RenderAttributes(w, n, DefinitionListAttributeFilter) + _, _ = w.WriteString(">\n") + } else { + _, _ = w.WriteString("<dl>\n") + } + } else { + _, _ = w.WriteString("</dl>\n") + } + return gast.WalkContinue, nil +} + +// DefinitionTermAttributeFilter defines attribute names which dd elements can have. +var DefinitionTermAttributeFilter = html.GlobalAttributeFilter + +func (r *DefinitionListHTMLRenderer) renderDefinitionTerm(w util.BufWriter, source []byte, n gast.Node, entering bool) (gast.WalkStatus, error) { + if entering { + if n.Attributes() != nil { + _, _ = w.WriteString("<dt") + html.RenderAttributes(w, n, DefinitionTermAttributeFilter) + _ = w.WriteByte('>') + } else { + _, _ = w.WriteString("<dt>") + } + } else { + _, _ = w.WriteString("</dt>\n") + } + return gast.WalkContinue, nil +} + +// DefinitionDescriptionAttributeFilter defines attribute names which dd elements can have. +var DefinitionDescriptionAttributeFilter = html.GlobalAttributeFilter + +func (r *DefinitionListHTMLRenderer) renderDefinitionDescription(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) { + if entering { + n := node.(*ast.DefinitionDescription) + _, _ = w.WriteString("<dd") + if n.Attributes() != nil { + html.RenderAttributes(w, n, DefinitionDescriptionAttributeFilter) + } + if n.IsTight { + _, _ = w.WriteString(">") + } else { + _, _ = w.WriteString(">\n") + } + } else { + _, _ = w.WriteString("</dd>\n") + } + return gast.WalkContinue, nil +} + +type definitionList struct { +} + +// DefinitionList is an extension that allow you to use PHP Markdown Extra Definition lists. +var DefinitionList = &definitionList{} + +func (e *definitionList) Extend(m goldmark.Markdown) { + m.Parser().AddOptions(parser.WithBlockParsers( + util.Prioritized(NewDefinitionListParser(), 101), + util.Prioritized(NewDefinitionDescriptionParser(), 102), + )) + m.Renderer().AddOptions(renderer.WithNodeRenderers( + util.Prioritized(NewDefinitionListHTMLRenderer(), 500), + )) +} diff --git a/vendor/github.com/yuin/goldmark/extension/footnote.go b/vendor/github.com/yuin/goldmark/extension/footnote.go new file mode 100644 index 00000000..d4552e5a --- /dev/null +++ b/vendor/github.com/yuin/goldmark/extension/footnote.go @@ -0,0 +1,670 @@ +package extension + +import ( + "bytes" + "strconv" + + "github.com/yuin/goldmark" + gast "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +var footnoteListKey = parser.NewContextKey() +var footnoteLinkListKey = parser.NewContextKey() + +type footnoteBlockParser struct { +} + +var defaultFootnoteBlockParser = &footnoteBlockParser{} + +// NewFootnoteBlockParser returns a new parser.BlockParser that can parse +// footnotes of the Markdown(PHP Markdown Extra) text. +func NewFootnoteBlockParser() parser.BlockParser { + return defaultFootnoteBlockParser +} + +func (b *footnoteBlockParser) Trigger() []byte { + return []byte{'['} +} + +func (b *footnoteBlockParser) Open(parent gast.Node, reader text.Reader, pc parser.Context) (gast.Node, parser.State) { + line, segment := reader.PeekLine() + pos := pc.BlockOffset() + if pos < 0 || line[pos] != '[' { + return nil, parser.NoChildren + } + pos++ + if pos > len(line)-1 || line[pos] != '^' { + return nil, parser.NoChildren + } + open := pos + 1 + closes := 0 + closure := util.FindClosure(line[pos+1:], '[', ']', false, false) + closes = pos + 1 + closure + next := closes + 1 + if closure > -1 { + if next >= len(line) || line[next] != ':' { + return nil, parser.NoChildren + } + } else { + return nil, parser.NoChildren + } + padding := segment.Padding + label := reader.Value(text.NewSegment(segment.Start+open-padding, segment.Start+closes-padding)) + if util.IsBlank(label) { + return nil, parser.NoChildren + } + item := ast.NewFootnote(label) + + pos = next + 1 - padding + if pos >= len(line) { + reader.Advance(pos) + return item, parser.NoChildren + } + reader.AdvanceAndSetPadding(pos, padding) + return item, parser.HasChildren +} + +func (b *footnoteBlockParser) Continue(node gast.Node, reader text.Reader, pc parser.Context) parser.State { + line, _ := reader.PeekLine() + if util.IsBlank(line) { + return parser.Continue | parser.HasChildren + } + childpos, padding := util.IndentPosition(line, reader.LineOffset(), 4) + if childpos < 0 { + return parser.Close + } + reader.AdvanceAndSetPadding(childpos, padding) + return parser.Continue | parser.HasChildren +} + +func (b *footnoteBlockParser) Close(node gast.Node, reader text.Reader, pc parser.Context) { + var list *ast.FootnoteList + if tlist := pc.Get(footnoteListKey); tlist != nil { + list = tlist.(*ast.FootnoteList) + } else { + list = ast.NewFootnoteList() + pc.Set(footnoteListKey, list) + node.Parent().InsertBefore(node.Parent(), node, list) + } + node.Parent().RemoveChild(node.Parent(), node) + list.AppendChild(list, node) +} + +func (b *footnoteBlockParser) CanInterruptParagraph() bool { + return true +} + +func (b *footnoteBlockParser) CanAcceptIndentedLine() bool { + return false +} + +type footnoteParser struct { +} + +var defaultFootnoteParser = &footnoteParser{} + +// NewFootnoteParser returns a new parser.InlineParser that can parse +// footnote links of the Markdown(PHP Markdown Extra) text. +func NewFootnoteParser() parser.InlineParser { + return defaultFootnoteParser +} + +func (s *footnoteParser) Trigger() []byte { + // footnote syntax probably conflict with the image syntax. + // So we need trigger this parser with '!'. + return []byte{'!', '['} +} + +func (s *footnoteParser) Parse(parent gast.Node, block text.Reader, pc parser.Context) gast.Node { + line, segment := block.PeekLine() + pos := 1 + if len(line) > 0 && line[0] == '!' { + pos++ + } + if pos >= len(line) || line[pos] != '^' { + return nil + } + pos++ + if pos >= len(line) { + return nil + } + open := pos + closure := util.FindClosure(line[pos:], '[', ']', false, false) + if closure < 0 { + return nil + } + closes := pos + closure + value := block.Value(text.NewSegment(segment.Start+open, segment.Start+closes)) + block.Advance(closes + 1) + + var list *ast.FootnoteList + if tlist := pc.Get(footnoteListKey); tlist != nil { + list = tlist.(*ast.FootnoteList) + } + if list == nil { + return nil + } + index := 0 + for def := list.FirstChild(); def != nil; def = def.NextSibling() { + d := def.(*ast.Footnote) + if bytes.Equal(d.Ref, value) { + if d.Index < 0 { + list.Count += 1 + d.Index = list.Count + } + index = d.Index + break + } + } + if index == 0 { + return nil + } + + fnlink := ast.NewFootnoteLink(index) + var fnlist []*ast.FootnoteLink + if tmp := pc.Get(footnoteLinkListKey); tmp != nil { + fnlist = tmp.([]*ast.FootnoteLink) + } else { + fnlist = []*ast.FootnoteLink{} + pc.Set(footnoteLinkListKey, fnlist) + } + pc.Set(footnoteLinkListKey, append(fnlist, fnlink)) + if line[0] == '!' { + parent.AppendChild(parent, gast.NewTextSegment(text.NewSegment(segment.Start, segment.Start+1))) + } + + return fnlink +} + +type footnoteASTTransformer struct { +} + +var defaultFootnoteASTTransformer = &footnoteASTTransformer{} + +// NewFootnoteASTTransformer returns a new parser.ASTTransformer that +// insert a footnote list to the last of the document. +func NewFootnoteASTTransformer() parser.ASTTransformer { + return defaultFootnoteASTTransformer +} + +func (a *footnoteASTTransformer) Transform(node *gast.Document, reader text.Reader, pc parser.Context) { + var list *ast.FootnoteList + var fnlist []*ast.FootnoteLink + if tmp := pc.Get(footnoteListKey); tmp != nil { + list = tmp.(*ast.FootnoteList) + } + if tmp := pc.Get(footnoteLinkListKey); tmp != nil { + fnlist = tmp.([]*ast.FootnoteLink) + } + + pc.Set(footnoteListKey, nil) + pc.Set(footnoteLinkListKey, nil) + + if list == nil { + return + } + + counter := map[int]int{} + if fnlist != nil { + for _, fnlink := range fnlist { + if fnlink.Index >= 0 { + counter[fnlink.Index]++ + } + } + for _, fnlink := range fnlist { + fnlink.RefCount = counter[fnlink.Index] + } + } + for footnote := list.FirstChild(); footnote != nil; { + var container gast.Node = footnote + next := footnote.NextSibling() + if fc := container.LastChild(); fc != nil && gast.IsParagraph(fc) { + container = fc + } + fn := footnote.(*ast.Footnote) + index := fn.Index + if index < 0 { + list.RemoveChild(list, footnote) + } else { + backLink := ast.NewFootnoteBacklink(index) + backLink.RefCount = counter[index] + container.AppendChild(container, backLink) + } + footnote = next + } + list.SortChildren(func(n1, n2 gast.Node) int { + if n1.(*ast.Footnote).Index < n2.(*ast.Footnote).Index { + return -1 + } + return 1 + }) + if list.Count <= 0 { + list.Parent().RemoveChild(list.Parent(), list) + return + } + + node.AppendChild(node, list) +} + +// FootnoteConfig holds configuration values for the footnote extension. +// +// Link* and Backlink* configurations have some variables: +// Occurrances of “^^” in the string will be replaced by the +// corresponding footnote number in the HTML output. +// Occurrances of “%%” will be replaced by a number for the +// reference (footnotes can have multiple references). +type FootnoteConfig struct { + html.Config + + // IDPrefix is a prefix for the id attributes generated by footnotes. + IDPrefix []byte + + // IDPrefix is a function that determines the id attribute for given Node. + IDPrefixFunction func(gast.Node) []byte + + // LinkTitle is an optional title attribute for footnote links. + LinkTitle []byte + + // BacklinkTitle is an optional title attribute for footnote backlinks. + BacklinkTitle []byte + + // LinkClass is a class for footnote links. + LinkClass []byte + + // BacklinkClass is a class for footnote backlinks. + BacklinkClass []byte + + // BacklinkHTML is an HTML content for footnote backlinks. + BacklinkHTML []byte +} + +// FootnoteOption interface is a functional option interface for the extension. +type FootnoteOption interface { + renderer.Option + // SetFootnoteOption sets given option to the extension. + SetFootnoteOption(*FootnoteConfig) +} + +// NewFootnoteConfig returns a new Config with defaults. +func NewFootnoteConfig() FootnoteConfig { + return FootnoteConfig{ + Config: html.NewConfig(), + LinkTitle: []byte(""), + BacklinkTitle: []byte(""), + LinkClass: []byte("footnote-ref"), + BacklinkClass: []byte("footnote-backref"), + BacklinkHTML: []byte("↩︎"), + } +} + +// SetOption implements renderer.SetOptioner. +func (c *FootnoteConfig) SetOption(name renderer.OptionName, value interface{}) { + switch name { + case optFootnoteIDPrefixFunction: + c.IDPrefixFunction = value.(func(gast.Node) []byte) + case optFootnoteIDPrefix: + c.IDPrefix = value.([]byte) + case optFootnoteLinkTitle: + c.LinkTitle = value.([]byte) + case optFootnoteBacklinkTitle: + c.BacklinkTitle = value.([]byte) + case optFootnoteLinkClass: + c.LinkClass = value.([]byte) + case optFootnoteBacklinkClass: + c.BacklinkClass = value.([]byte) + case optFootnoteBacklinkHTML: + c.BacklinkHTML = value.([]byte) + default: + c.Config.SetOption(name, value) + } +} + +type withFootnoteHTMLOptions struct { + value []html.Option +} + +func (o *withFootnoteHTMLOptions) SetConfig(c *renderer.Config) { + if o.value != nil { + for _, v := range o.value { + v.(renderer.Option).SetConfig(c) + } + } +} + +func (o *withFootnoteHTMLOptions) SetFootnoteOption(c *FootnoteConfig) { + if o.value != nil { + for _, v := range o.value { + v.SetHTMLOption(&c.Config) + } + } +} + +// WithFootnoteHTMLOptions is functional option that wraps goldmark HTMLRenderer options. +func WithFootnoteHTMLOptions(opts ...html.Option) FootnoteOption { + return &withFootnoteHTMLOptions{opts} +} + +const optFootnoteIDPrefix renderer.OptionName = "FootnoteIDPrefix" + +type withFootnoteIDPrefix struct { + value []byte +} + +func (o *withFootnoteIDPrefix) SetConfig(c *renderer.Config) { + c.Options[optFootnoteIDPrefix] = o.value +} + +func (o *withFootnoteIDPrefix) SetFootnoteOption(c *FootnoteConfig) { + c.IDPrefix = o.value +} + +// WithFootnoteIDPrefix is a functional option that is a prefix for the id attributes generated by footnotes. +func WithFootnoteIDPrefix(a []byte) FootnoteOption { + return &withFootnoteIDPrefix{a} +} + +const optFootnoteIDPrefixFunction renderer.OptionName = "FootnoteIDPrefixFunction" + +type withFootnoteIDPrefixFunction struct { + value func(gast.Node) []byte +} + +func (o *withFootnoteIDPrefixFunction) SetConfig(c *renderer.Config) { + c.Options[optFootnoteIDPrefixFunction] = o.value +} + +func (o *withFootnoteIDPrefixFunction) SetFootnoteOption(c *FootnoteConfig) { + c.IDPrefixFunction = o.value +} + +// WithFootnoteIDPrefixFunction is a functional option that is a prefix for the id attributes generated by footnotes. +func WithFootnoteIDPrefixFunction(a func(gast.Node) []byte) FootnoteOption { + return &withFootnoteIDPrefixFunction{a} +} + +const optFootnoteLinkTitle renderer.OptionName = "FootnoteLinkTitle" + +type withFootnoteLinkTitle struct { + value []byte +} + +func (o *withFootnoteLinkTitle) SetConfig(c *renderer.Config) { + c.Options[optFootnoteLinkTitle] = o.value +} + +func (o *withFootnoteLinkTitle) SetFootnoteOption(c *FootnoteConfig) { + c.LinkTitle = o.value +} + +// WithFootnoteLinkTitle is a functional option that is an optional title attribute for footnote links. +func WithFootnoteLinkTitle(a []byte) FootnoteOption { + return &withFootnoteLinkTitle{a} +} + +const optFootnoteBacklinkTitle renderer.OptionName = "FootnoteBacklinkTitle" + +type withFootnoteBacklinkTitle struct { + value []byte +} + +func (o *withFootnoteBacklinkTitle) SetConfig(c *renderer.Config) { + c.Options[optFootnoteBacklinkTitle] = o.value +} + +func (o *withFootnoteBacklinkTitle) SetFootnoteOption(c *FootnoteConfig) { + c.BacklinkTitle = o.value +} + +// WithFootnoteBacklinkTitle is a functional option that is an optional title attribute for footnote backlinks. +func WithFootnoteBacklinkTitle(a []byte) FootnoteOption { + return &withFootnoteBacklinkTitle{a} +} + +const optFootnoteLinkClass renderer.OptionName = "FootnoteLinkClass" + +type withFootnoteLinkClass struct { + value []byte +} + +func (o *withFootnoteLinkClass) SetConfig(c *renderer.Config) { + c.Options[optFootnoteLinkClass] = o.value +} + +func (o *withFootnoteLinkClass) SetFootnoteOption(c *FootnoteConfig) { + c.LinkClass = o.value +} + +// WithFootnoteLinkClass is a functional option that is a class for footnote links. +func WithFootnoteLinkClass(a []byte) FootnoteOption { + return &withFootnoteLinkClass{a} +} + +const optFootnoteBacklinkClass renderer.OptionName = "FootnoteBacklinkClass" + +type withFootnoteBacklinkClass struct { + value []byte +} + +func (o *withFootnoteBacklinkClass) SetConfig(c *renderer.Config) { + c.Options[optFootnoteBacklinkClass] = o.value +} + +func (o *withFootnoteBacklinkClass) SetFootnoteOption(c *FootnoteConfig) { + c.BacklinkClass = o.value +} + +// WithFootnoteBacklinkClass is a functional option that is a class for footnote backlinks. +func WithFootnoteBacklinkClass(a []byte) FootnoteOption { + return &withFootnoteBacklinkClass{a} +} + +const optFootnoteBacklinkHTML renderer.OptionName = "FootnoteBacklinkHTML" + +type withFootnoteBacklinkHTML struct { + value []byte +} + +func (o *withFootnoteBacklinkHTML) SetConfig(c *renderer.Config) { + c.Options[optFootnoteBacklinkHTML] = o.value +} + +func (o *withFootnoteBacklinkHTML) SetFootnoteOption(c *FootnoteConfig) { + c.BacklinkHTML = o.value +} + +// WithFootnoteBacklinkHTML is an HTML content for footnote backlinks. +func WithFootnoteBacklinkHTML(a []byte) FootnoteOption { + return &withFootnoteBacklinkHTML{a} +} + +// FootnoteHTMLRenderer is a renderer.NodeRenderer implementation that +// renders FootnoteLink nodes. +type FootnoteHTMLRenderer struct { + FootnoteConfig +} + +// NewFootnoteHTMLRenderer returns a new FootnoteHTMLRenderer. +func NewFootnoteHTMLRenderer(opts ...FootnoteOption) renderer.NodeRenderer { + r := &FootnoteHTMLRenderer{ + FootnoteConfig: NewFootnoteConfig(), + } + for _, opt := range opts { + opt.SetFootnoteOption(&r.FootnoteConfig) + } + return r +} + +// RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs. +func (r *FootnoteHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(ast.KindFootnoteLink, r.renderFootnoteLink) + reg.Register(ast.KindFootnoteBacklink, r.renderFootnoteBacklink) + reg.Register(ast.KindFootnote, r.renderFootnote) + reg.Register(ast.KindFootnoteList, r.renderFootnoteList) +} + +func (r *FootnoteHTMLRenderer) renderFootnoteLink(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) { + if entering { + n := node.(*ast.FootnoteLink) + is := strconv.Itoa(n.Index) + _, _ = w.WriteString(`<sup id="`) + _, _ = w.Write(r.idPrefix(node)) + _, _ = w.WriteString(`fnref:`) + _, _ = w.WriteString(is) + _, _ = w.WriteString(`"><a href="#`) + _, _ = w.Write(r.idPrefix(node)) + _, _ = w.WriteString(`fn:`) + _, _ = w.WriteString(is) + _, _ = w.WriteString(`" class="`) + _, _ = w.Write(applyFootnoteTemplate(r.FootnoteConfig.LinkClass, + n.Index, n.RefCount)) + if len(r.FootnoteConfig.LinkTitle) > 0 { + _, _ = w.WriteString(`" title="`) + _, _ = w.Write(util.EscapeHTML(applyFootnoteTemplate(r.FootnoteConfig.LinkTitle, n.Index, n.RefCount))) + } + _, _ = w.WriteString(`" role="doc-noteref">`) + + _, _ = w.WriteString(is) + _, _ = w.WriteString(`</a></sup>`) + } + return gast.WalkContinue, nil +} + +func (r *FootnoteHTMLRenderer) renderFootnoteBacklink(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) { + if entering { + n := node.(*ast.FootnoteBacklink) + is := strconv.Itoa(n.Index) + _, _ = w.WriteString(` <a href="#`) + _, _ = w.Write(r.idPrefix(node)) + _, _ = w.WriteString(`fnref:`) + _, _ = w.WriteString(is) + _, _ = w.WriteString(`" class="`) + _, _ = w.Write(applyFootnoteTemplate(r.FootnoteConfig.BacklinkClass, n.Index, n.RefCount)) + if len(r.FootnoteConfig.BacklinkTitle) > 0 { + _, _ = w.WriteString(`" title="`) + _, _ = w.Write(util.EscapeHTML(applyFootnoteTemplate(r.FootnoteConfig.BacklinkTitle, n.Index, n.RefCount))) + } + _, _ = w.WriteString(`" role="doc-backlink">`) + _, _ = w.Write(applyFootnoteTemplate(r.FootnoteConfig.BacklinkHTML, n.Index, n.RefCount)) + _, _ = w.WriteString(`</a>`) + } + return gast.WalkContinue, nil +} + +func (r *FootnoteHTMLRenderer) renderFootnote(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) { + n := node.(*ast.Footnote) + is := strconv.Itoa(n.Index) + if entering { + _, _ = w.WriteString(`<li id="`) + _, _ = w.Write(r.idPrefix(node)) + _, _ = w.WriteString(`fn:`) + _, _ = w.WriteString(is) + _, _ = w.WriteString(`" role="doc-endnote"`) + if node.Attributes() != nil { + html.RenderAttributes(w, node, html.ListItemAttributeFilter) + } + _, _ = w.WriteString(">\n") + } else { + _, _ = w.WriteString("</li>\n") + } + return gast.WalkContinue, nil +} + +func (r *FootnoteHTMLRenderer) renderFootnoteList(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) { + tag := "section" + if r.Config.XHTML { + tag = "div" + } + if entering { + _, _ = w.WriteString("<") + _, _ = w.WriteString(tag) + _, _ = w.WriteString(` class="footnotes" role="doc-endnotes"`) + if node.Attributes() != nil { + html.RenderAttributes(w, node, html.GlobalAttributeFilter) + } + _ = w.WriteByte('>') + if r.Config.XHTML { + _, _ = w.WriteString("\n<hr />\n") + } else { + _, _ = w.WriteString("\n<hr>\n") + } + _, _ = w.WriteString("<ol>\n") + } else { + _, _ = w.WriteString("</ol>\n") + _, _ = w.WriteString("</") + _, _ = w.WriteString(tag) + _, _ = w.WriteString(">\n") + } + return gast.WalkContinue, nil +} + +func (r *FootnoteHTMLRenderer) idPrefix(node gast.Node) []byte { + if r.FootnoteConfig.IDPrefix != nil { + return r.FootnoteConfig.IDPrefix + } + if r.FootnoteConfig.IDPrefixFunction != nil { + return r.FootnoteConfig.IDPrefixFunction(node) + } + return []byte("") +} + +func applyFootnoteTemplate(b []byte, index, refCount int) []byte { + fast := true + for i, c := range b { + if i != 0 { + if b[i-1] == '^' && c == '^' { + fast = false + break + } + if b[i-1] == '%' && c == '%' { + fast = false + break + } + } + } + if fast { + return b + } + is := []byte(strconv.Itoa(index)) + rs := []byte(strconv.Itoa(refCount)) + ret := bytes.Replace(b, []byte("^^"), is, -1) + return bytes.Replace(ret, []byte("%%"), rs, -1) +} + +type footnote struct { + options []FootnoteOption +} + +// Footnote is an extension that allow you to use PHP Markdown Extra Footnotes. +var Footnote = &footnote{ + options: []FootnoteOption{}, +} + +// NewFootnote returns a new extension with given options. +func NewFootnote(opts ...FootnoteOption) goldmark.Extender { + return &footnote{ + options: opts, + } +} + +func (e *footnote) Extend(m goldmark.Markdown) { + m.Parser().AddOptions( + parser.WithBlockParsers( + util.Prioritized(NewFootnoteBlockParser(), 999), + ), + parser.WithInlineParsers( + util.Prioritized(NewFootnoteParser(), 101), + ), + parser.WithASTTransformers( + util.Prioritized(NewFootnoteASTTransformer(), 999), + ), + ) + m.Renderer().AddOptions(renderer.WithNodeRenderers( + util.Prioritized(NewFootnoteHTMLRenderer(e.options...), 500), + )) +} diff --git a/vendor/github.com/yuin/goldmark/extension/gfm.go b/vendor/github.com/yuin/goldmark/extension/gfm.go new file mode 100644 index 00000000..a570fbdb --- /dev/null +++ b/vendor/github.com/yuin/goldmark/extension/gfm.go @@ -0,0 +1,18 @@ +package extension + +import ( + "github.com/yuin/goldmark" +) + +type gfm struct { +} + +// GFM is an extension that provides Github Flavored markdown functionalities. +var GFM = &gfm{} + +func (e *gfm) Extend(m goldmark.Markdown) { + Linkify.Extend(m) + Table.Extend(m) + Strikethrough.Extend(m) + TaskList.Extend(m) +} diff --git a/vendor/github.com/yuin/goldmark/extension/linkify.go b/vendor/github.com/yuin/goldmark/extension/linkify.go new file mode 100644 index 00000000..2f046eb5 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/extension/linkify.go @@ -0,0 +1,318 @@ +package extension + +import ( + "bytes" + "regexp" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +var wwwURLRegxp = regexp.MustCompile(`^www\.[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-z]+(?:[/#?][-a-zA-Z0-9@:%_\+.~#!?&/=\(\);,'">\^{}\[\]` + "`" + `]*)?`) + +var urlRegexp = regexp.MustCompile(`^(?:http|https|ftp)://[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-z]+(?::\d+)?(?:[/#?][-a-zA-Z0-9@:%_+.~#$!?&/=\(\);,'">\^{}\[\]` + "`" + `]*)?`) + +// An LinkifyConfig struct is a data structure that holds configuration of the +// Linkify extension. +type LinkifyConfig struct { + AllowedProtocols [][]byte + URLRegexp *regexp.Regexp + WWWRegexp *regexp.Regexp + EmailRegexp *regexp.Regexp +} + +const ( + optLinkifyAllowedProtocols parser.OptionName = "LinkifyAllowedProtocols" + optLinkifyURLRegexp parser.OptionName = "LinkifyURLRegexp" + optLinkifyWWWRegexp parser.OptionName = "LinkifyWWWRegexp" + optLinkifyEmailRegexp parser.OptionName = "LinkifyEmailRegexp" +) + +// SetOption implements SetOptioner. +func (c *LinkifyConfig) SetOption(name parser.OptionName, value interface{}) { + switch name { + case optLinkifyAllowedProtocols: + c.AllowedProtocols = value.([][]byte) + case optLinkifyURLRegexp: + c.URLRegexp = value.(*regexp.Regexp) + case optLinkifyWWWRegexp: + c.WWWRegexp = value.(*regexp.Regexp) + case optLinkifyEmailRegexp: + c.EmailRegexp = value.(*regexp.Regexp) + } +} + +// A LinkifyOption interface sets options for the LinkifyOption. +type LinkifyOption interface { + parser.Option + SetLinkifyOption(*LinkifyConfig) +} + +type withLinkifyAllowedProtocols struct { + value [][]byte +} + +func (o *withLinkifyAllowedProtocols) SetParserOption(c *parser.Config) { + c.Options[optLinkifyAllowedProtocols] = o.value +} + +func (o *withLinkifyAllowedProtocols) SetLinkifyOption(p *LinkifyConfig) { + p.AllowedProtocols = o.value +} + +// WithLinkifyAllowedProtocols is a functional option that specify allowed +// protocols in autolinks. Each protocol must end with ':' like +// 'http:' . +func WithLinkifyAllowedProtocols(value [][]byte) LinkifyOption { + return &withLinkifyAllowedProtocols{ + value: value, + } +} + +type withLinkifyURLRegexp struct { + value *regexp.Regexp +} + +func (o *withLinkifyURLRegexp) SetParserOption(c *parser.Config) { + c.Options[optLinkifyURLRegexp] = o.value +} + +func (o *withLinkifyURLRegexp) SetLinkifyOption(p *LinkifyConfig) { + p.URLRegexp = o.value +} + +// WithLinkifyURLRegexp is a functional option that specify +// a pattern of the URL including a protocol. +func WithLinkifyURLRegexp(value *regexp.Regexp) LinkifyOption { + return &withLinkifyURLRegexp{ + value: value, + } +} + +// WithLinkifyWWWRegexp is a functional option that specify +// a pattern of the URL without a protocol. +// This pattern must start with 'www.' . +type withLinkifyWWWRegexp struct { + value *regexp.Regexp +} + +func (o *withLinkifyWWWRegexp) SetParserOption(c *parser.Config) { + c.Options[optLinkifyWWWRegexp] = o.value +} + +func (o *withLinkifyWWWRegexp) SetLinkifyOption(p *LinkifyConfig) { + p.WWWRegexp = o.value +} + +func WithLinkifyWWWRegexp(value *regexp.Regexp) LinkifyOption { + return &withLinkifyWWWRegexp{ + value: value, + } +} + +// WithLinkifyWWWRegexp is a functional otpion that specify +// a pattern of the email address. +type withLinkifyEmailRegexp struct { + value *regexp.Regexp +} + +func (o *withLinkifyEmailRegexp) SetParserOption(c *parser.Config) { + c.Options[optLinkifyEmailRegexp] = o.value +} + +func (o *withLinkifyEmailRegexp) SetLinkifyOption(p *LinkifyConfig) { + p.EmailRegexp = o.value +} + +func WithLinkifyEmailRegexp(value *regexp.Regexp) LinkifyOption { + return &withLinkifyEmailRegexp{ + value: value, + } +} + +type linkifyParser struct { + LinkifyConfig +} + +// NewLinkifyParser return a new InlineParser can parse +// text that seems like a URL. +func NewLinkifyParser(opts ...LinkifyOption) parser.InlineParser { + p := &linkifyParser{ + LinkifyConfig: LinkifyConfig{ + AllowedProtocols: nil, + URLRegexp: urlRegexp, + WWWRegexp: wwwURLRegxp, + }, + } + for _, o := range opts { + o.SetLinkifyOption(&p.LinkifyConfig) + } + return p +} + +func (s *linkifyParser) Trigger() []byte { + // ' ' indicates any white spaces and a line head + return []byte{' ', '*', '_', '~', '('} +} + +var ( + protoHTTP = []byte("http:") + protoHTTPS = []byte("https:") + protoFTP = []byte("ftp:") + domainWWW = []byte("www.") +) + +func (s *linkifyParser) Parse(parent ast.Node, block text.Reader, pc parser.Context) ast.Node { + if pc.IsInLinkLabel() { + return nil + } + line, segment := block.PeekLine() + consumes := 0 + start := segment.Start + c := line[0] + // advance if current position is not a line head. + if c == ' ' || c == '*' || c == '_' || c == '~' || c == '(' { + consumes++ + start++ + line = line[1:] + } + + var m []int + var protocol []byte + var typ ast.AutoLinkType = ast.AutoLinkURL + if s.LinkifyConfig.AllowedProtocols == nil { + if bytes.HasPrefix(line, protoHTTP) || bytes.HasPrefix(line, protoHTTPS) || bytes.HasPrefix(line, protoFTP) { + m = s.LinkifyConfig.URLRegexp.FindSubmatchIndex(line) + } + } else { + for _, prefix := range s.LinkifyConfig.AllowedProtocols { + if bytes.HasPrefix(line, prefix) { + m = s.LinkifyConfig.URLRegexp.FindSubmatchIndex(line) + break + } + } + } + if m == nil && bytes.HasPrefix(line, domainWWW) { + m = s.LinkifyConfig.WWWRegexp.FindSubmatchIndex(line) + protocol = []byte("http") + } + if m != nil && m[0] != 0 { + m = nil + } + if m != nil && m[0] == 0 { + lastChar := line[m[1]-1] + if lastChar == '.' { + m[1]-- + } else if lastChar == ')' { + closing := 0 + for i := m[1] - 1; i >= m[0]; i-- { + if line[i] == ')' { + closing++ + } else if line[i] == '(' { + closing-- + } + } + if closing > 0 { + m[1] -= closing + } + } else if lastChar == ';' { + i := m[1] - 2 + for ; i >= m[0]; i-- { + if util.IsAlphaNumeric(line[i]) { + continue + } + break + } + if i != m[1]-2 { + if line[i] == '&' { + m[1] -= m[1] - i + } + } + } + } + if m == nil { + if len(line) > 0 && util.IsPunct(line[0]) { + return nil + } + typ = ast.AutoLinkEmail + stop := -1 + if s.LinkifyConfig.EmailRegexp == nil { + stop = util.FindEmailIndex(line) + } else { + m := s.LinkifyConfig.EmailRegexp.FindSubmatchIndex(line) + if m != nil && m[0] == 0 { + stop = m[1] + } + } + if stop < 0 { + return nil + } + at := bytes.IndexByte(line, '@') + m = []int{0, stop, at, stop - 1} + if m == nil || bytes.IndexByte(line[m[2]:m[3]], '.') < 0 { + return nil + } + lastChar := line[m[1]-1] + if lastChar == '.' { + m[1]-- + } + if m[1] < len(line) { + nextChar := line[m[1]] + if nextChar == '-' || nextChar == '_' { + return nil + } + } + } + if m == nil { + return nil + } + if consumes != 0 { + s := segment.WithStop(segment.Start + 1) + ast.MergeOrAppendTextSegment(parent, s) + } + i := m[1] - 1 + for ; i > 0; i-- { + c := line[i] + switch c { + case '?', '!', '.', ',', ':', '*', '_', '~': + default: + goto endfor + } + } +endfor: + i++ + consumes += i + block.Advance(consumes) + n := ast.NewTextSegment(text.NewSegment(start, start+i)) + link := ast.NewAutoLink(typ, n) + link.Protocol = protocol + return link +} + +func (s *linkifyParser) CloseBlock(parent ast.Node, pc parser.Context) { + // nothing to do +} + +type linkify struct { + options []LinkifyOption +} + +// Linkify is an extension that allow you to parse text that seems like a URL. +var Linkify = &linkify{} + +func NewLinkify(opts ...LinkifyOption) goldmark.Extender { + return &linkify{ + options: opts, + } +} + +func (e *linkify) Extend(m goldmark.Markdown) { + m.Parser().AddOptions( + parser.WithInlineParsers( + util.Prioritized(NewLinkifyParser(e.options...), 999), + ), + ) +} diff --git a/vendor/github.com/yuin/goldmark/extension/strikethrough.go b/vendor/github.com/yuin/goldmark/extension/strikethrough.go new file mode 100644 index 00000000..1b629ad8 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/extension/strikethrough.go @@ -0,0 +1,116 @@ +package extension + +import ( + "github.com/yuin/goldmark" + gast "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +type strikethroughDelimiterProcessor struct { +} + +func (p *strikethroughDelimiterProcessor) IsDelimiter(b byte) bool { + return b == '~' +} + +func (p *strikethroughDelimiterProcessor) CanOpenCloser(opener, closer *parser.Delimiter) bool { + return opener.Char == closer.Char +} + +func (p *strikethroughDelimiterProcessor) OnMatch(consumes int) gast.Node { + return ast.NewStrikethrough() +} + +var defaultStrikethroughDelimiterProcessor = &strikethroughDelimiterProcessor{} + +type strikethroughParser struct { +} + +var defaultStrikethroughParser = &strikethroughParser{} + +// NewStrikethroughParser return a new InlineParser that parses +// strikethrough expressions. +func NewStrikethroughParser() parser.InlineParser { + return defaultStrikethroughParser +} + +func (s *strikethroughParser) Trigger() []byte { + return []byte{'~'} +} + +func (s *strikethroughParser) Parse(parent gast.Node, block text.Reader, pc parser.Context) gast.Node { + before := block.PrecendingCharacter() + line, segment := block.PeekLine() + node := parser.ScanDelimiter(line, before, 2, defaultStrikethroughDelimiterProcessor) + if node == nil { + return nil + } + node.Segment = segment.WithStop(segment.Start + node.OriginalLength) + block.Advance(node.OriginalLength) + pc.PushDelimiter(node) + return node +} + +func (s *strikethroughParser) CloseBlock(parent gast.Node, pc parser.Context) { + // nothing to do +} + +// StrikethroughHTMLRenderer is a renderer.NodeRenderer implementation that +// renders Strikethrough nodes. +type StrikethroughHTMLRenderer struct { + html.Config +} + +// NewStrikethroughHTMLRenderer returns a new StrikethroughHTMLRenderer. +func NewStrikethroughHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { + r := &StrikethroughHTMLRenderer{ + Config: html.NewConfig(), + } + for _, opt := range opts { + opt.SetHTMLOption(&r.Config) + } + return r +} + +// RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs. +func (r *StrikethroughHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(ast.KindStrikethrough, r.renderStrikethrough) +} + +// StrikethroughAttributeFilter defines attribute names which dd elements can have. +var StrikethroughAttributeFilter = html.GlobalAttributeFilter + +func (r *StrikethroughHTMLRenderer) renderStrikethrough(w util.BufWriter, source []byte, n gast.Node, entering bool) (gast.WalkStatus, error) { + if entering { + if n.Attributes() != nil { + _, _ = w.WriteString("<del") + html.RenderAttributes(w, n, StrikethroughAttributeFilter) + _ = w.WriteByte('>') + } else { + _, _ = w.WriteString("<del>") + } + } else { + _, _ = w.WriteString("</del>") + } + return gast.WalkContinue, nil +} + +type strikethrough struct { +} + +// Strikethrough is an extension that allow you to use strikethrough expression like '~~text~~' . +var Strikethrough = &strikethrough{} + +func (e *strikethrough) Extend(m goldmark.Markdown) { + m.Parser().AddOptions(parser.WithInlineParsers( + util.Prioritized(NewStrikethroughParser(), 500), + )) + m.Renderer().AddOptions(renderer.WithNodeRenderers( + util.Prioritized(NewStrikethroughHTMLRenderer(), 500), + )) +} diff --git a/vendor/github.com/yuin/goldmark/extension/table.go b/vendor/github.com/yuin/goldmark/extension/table.go new file mode 100644 index 00000000..c637b99f --- /dev/null +++ b/vendor/github.com/yuin/goldmark/extension/table.go @@ -0,0 +1,552 @@ +package extension + +import ( + "bytes" + "fmt" + "regexp" + + "github.com/yuin/goldmark" + gast "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +var escapedPipeCellListKey = parser.NewContextKey() + +type escapedPipeCell struct { + Cell *ast.TableCell + Pos []int + Transformed bool +} + +// TableCellAlignMethod indicates how are table cells aligned in HTML format.indicates how are table cells aligned in HTML format. +type TableCellAlignMethod int + +const ( + // TableCellAlignDefault renders alignments by default method. + // With XHTML, alignments are rendered as an align attribute. + // With HTML5, alignments are rendered as a style attribute. + TableCellAlignDefault TableCellAlignMethod = iota + + // TableCellAlignAttribute renders alignments as an align attribute. + TableCellAlignAttribute + + // TableCellAlignStyle renders alignments as a style attribute. + TableCellAlignStyle + + // TableCellAlignNone does not care about alignments. + // If you using classes or other styles, you can add these attributes + // in an ASTTransformer. + TableCellAlignNone +) + +// TableConfig struct holds options for the extension. +type TableConfig struct { + html.Config + + // TableCellAlignMethod indicates how are table celss aligned. + TableCellAlignMethod TableCellAlignMethod +} + +// TableOption interface is a functional option interface for the extension. +type TableOption interface { + renderer.Option + // SetTableOption sets given option to the extension. + SetTableOption(*TableConfig) +} + +// NewTableConfig returns a new Config with defaults. +func NewTableConfig() TableConfig { + return TableConfig{ + Config: html.NewConfig(), + TableCellAlignMethod: TableCellAlignDefault, + } +} + +// SetOption implements renderer.SetOptioner. +func (c *TableConfig) SetOption(name renderer.OptionName, value interface{}) { + switch name { + case optTableCellAlignMethod: + c.TableCellAlignMethod = value.(TableCellAlignMethod) + default: + c.Config.SetOption(name, value) + } +} + +type withTableHTMLOptions struct { + value []html.Option +} + +func (o *withTableHTMLOptions) SetConfig(c *renderer.Config) { + if o.value != nil { + for _, v := range o.value { + v.(renderer.Option).SetConfig(c) + } + } +} + +func (o *withTableHTMLOptions) SetTableOption(c *TableConfig) { + if o.value != nil { + for _, v := range o.value { + v.SetHTMLOption(&c.Config) + } + } +} + +// WithTableHTMLOptions is functional option that wraps goldmark HTMLRenderer options. +func WithTableHTMLOptions(opts ...html.Option) TableOption { + return &withTableHTMLOptions{opts} +} + +const optTableCellAlignMethod renderer.OptionName = "TableTableCellAlignMethod" + +type withTableCellAlignMethod struct { + value TableCellAlignMethod +} + +func (o *withTableCellAlignMethod) SetConfig(c *renderer.Config) { + c.Options[optTableCellAlignMethod] = o.value +} + +func (o *withTableCellAlignMethod) SetTableOption(c *TableConfig) { + c.TableCellAlignMethod = o.value +} + +// WithTableCellAlignMethod is a functional option that indicates how are table cells aligned in HTML format. +func WithTableCellAlignMethod(a TableCellAlignMethod) TableOption { + return &withTableCellAlignMethod{a} +} + +func isTableDelim(bs []byte) bool { + for _, b := range bs { + if !(util.IsSpace(b) || b == '-' || b == '|' || b == ':') { + return false + } + } + return true +} + +var tableDelimLeft = regexp.MustCompile(`^\s*\:\-+\s*$`) +var tableDelimRight = regexp.MustCompile(`^\s*\-+\:\s*$`) +var tableDelimCenter = regexp.MustCompile(`^\s*\:\-+\:\s*$`) +var tableDelimNone = regexp.MustCompile(`^\s*\-+\s*$`) + +type tableParagraphTransformer struct { +} + +var defaultTableParagraphTransformer = &tableParagraphTransformer{} + +// NewTableParagraphTransformer returns a new ParagraphTransformer +// that can transform paragraphs into tables. +func NewTableParagraphTransformer() parser.ParagraphTransformer { + return defaultTableParagraphTransformer +} + +func (b *tableParagraphTransformer) Transform(node *gast.Paragraph, reader text.Reader, pc parser.Context) { + lines := node.Lines() + if lines.Len() < 2 { + return + } + for i := 1; i < lines.Len(); i++ { + alignments := b.parseDelimiter(lines.At(i), reader) + if alignments == nil { + continue + } + header := b.parseRow(lines.At(i-1), alignments, true, reader, pc) + if header == nil || len(alignments) != header.ChildCount() { + return + } + table := ast.NewTable() + table.Alignments = alignments + table.AppendChild(table, ast.NewTableHeader(header)) + for j := i + 1; j < lines.Len(); j++ { + table.AppendChild(table, b.parseRow(lines.At(j), alignments, false, reader, pc)) + } + node.Lines().SetSliced(0, i-1) + node.Parent().InsertAfter(node.Parent(), node, table) + if node.Lines().Len() == 0 { + node.Parent().RemoveChild(node.Parent(), node) + } else { + last := node.Lines().At(i - 2) + last.Stop = last.Stop - 1 // trim last newline(\n) + node.Lines().Set(i-2, last) + } + } +} + +func (b *tableParagraphTransformer) parseRow(segment text.Segment, alignments []ast.Alignment, isHeader bool, reader text.Reader, pc parser.Context) *ast.TableRow { + source := reader.Source() + line := segment.Value(source) + pos := 0 + pos += util.TrimLeftSpaceLength(line) + limit := len(line) + limit -= util.TrimRightSpaceLength(line) + row := ast.NewTableRow(alignments) + if len(line) > 0 && line[pos] == '|' { + pos++ + } + if len(line) > 0 && line[limit-1] == '|' { + limit-- + } + i := 0 + for ; pos < limit; i++ { + alignment := ast.AlignNone + if i >= len(alignments) { + if !isHeader { + return row + } + } else { + alignment = alignments[i] + } + + var escapedCell *escapedPipeCell + node := ast.NewTableCell() + node.Alignment = alignment + hasBacktick := false + closure := pos + for ; closure < limit; closure++ { + if line[closure] == '`' { + hasBacktick = true + } + if line[closure] == '|' { + if closure == 0 || line[closure-1] != '\\' { + break + } else if hasBacktick { + if escapedCell == nil { + escapedCell = &escapedPipeCell{node, []int{}, false} + escapedList := pc.ComputeIfAbsent(escapedPipeCellListKey, + func() interface{} { + return []*escapedPipeCell{} + }).([]*escapedPipeCell) + escapedList = append(escapedList, escapedCell) + pc.Set(escapedPipeCellListKey, escapedList) + } + escapedCell.Pos = append(escapedCell.Pos, segment.Start+closure-1) + } + } + } + seg := text.NewSegment(segment.Start+pos, segment.Start+closure) + seg = seg.TrimLeftSpace(source) + seg = seg.TrimRightSpace(source) + node.Lines().Append(seg) + row.AppendChild(row, node) + pos = closure + 1 + } + for ; i < len(alignments); i++ { + row.AppendChild(row, ast.NewTableCell()) + } + return row +} + +func (b *tableParagraphTransformer) parseDelimiter(segment text.Segment, reader text.Reader) []ast.Alignment { + line := segment.Value(reader.Source()) + if !isTableDelim(line) { + return nil + } + cols := bytes.Split(line, []byte{'|'}) + if util.IsBlank(cols[0]) { + cols = cols[1:] + } + if len(cols) > 0 && util.IsBlank(cols[len(cols)-1]) { + cols = cols[:len(cols)-1] + } + + var alignments []ast.Alignment + for _, col := range cols { + if tableDelimLeft.Match(col) { + alignments = append(alignments, ast.AlignLeft) + } else if tableDelimRight.Match(col) { + alignments = append(alignments, ast.AlignRight) + } else if tableDelimCenter.Match(col) { + alignments = append(alignments, ast.AlignCenter) + } else if tableDelimNone.Match(col) { + alignments = append(alignments, ast.AlignNone) + } else { + return nil + } + } + return alignments +} + +type tableASTTransformer struct { +} + +var defaultTableASTTransformer = &tableASTTransformer{} + +// NewTableASTTransformer returns a parser.ASTTransformer for tables. +func NewTableASTTransformer() parser.ASTTransformer { + return defaultTableASTTransformer +} + +func (a *tableASTTransformer) Transform(node *gast.Document, reader text.Reader, pc parser.Context) { + lst := pc.Get(escapedPipeCellListKey) + if lst == nil { + return + } + pc.Set(escapedPipeCellListKey, nil) + for _, v := range lst.([]*escapedPipeCell) { + if v.Transformed { + continue + } + _ = gast.Walk(v.Cell, func(n gast.Node, entering bool) (gast.WalkStatus, error) { + if !entering || n.Kind() != gast.KindCodeSpan { + return gast.WalkContinue, nil + } + + for c := n.FirstChild(); c != nil; { + next := c.NextSibling() + if c.Kind() != gast.KindText { + c = next + continue + } + parent := c.Parent() + ts := &c.(*gast.Text).Segment + n := c + for _, v := range lst.([]*escapedPipeCell) { + for _, pos := range v.Pos { + if ts.Start <= pos && pos < ts.Stop { + segment := n.(*gast.Text).Segment + n1 := gast.NewRawTextSegment(segment.WithStop(pos)) + n2 := gast.NewRawTextSegment(segment.WithStart(pos + 1)) + parent.InsertAfter(parent, n, n1) + parent.InsertAfter(parent, n1, n2) + parent.RemoveChild(parent, n) + n = n2 + v.Transformed = true + } + } + } + c = next + } + return gast.WalkContinue, nil + }) + } +} + +// TableHTMLRenderer is a renderer.NodeRenderer implementation that +// renders Table nodes. +type TableHTMLRenderer struct { + TableConfig +} + +// NewTableHTMLRenderer returns a new TableHTMLRenderer. +func NewTableHTMLRenderer(opts ...TableOption) renderer.NodeRenderer { + r := &TableHTMLRenderer{ + TableConfig: NewTableConfig(), + } + for _, opt := range opts { + opt.SetTableOption(&r.TableConfig) + } + return r +} + +// RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs. +func (r *TableHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(ast.KindTable, r.renderTable) + reg.Register(ast.KindTableHeader, r.renderTableHeader) + reg.Register(ast.KindTableRow, r.renderTableRow) + reg.Register(ast.KindTableCell, r.renderTableCell) +} + +// TableAttributeFilter defines attribute names which table elements can have. +var TableAttributeFilter = html.GlobalAttributeFilter.Extend( + []byte("align"), // [Deprecated] + []byte("bgcolor"), // [Deprecated] + []byte("border"), // [Deprecated] + []byte("cellpadding"), // [Deprecated] + []byte("cellspacing"), // [Deprecated] + []byte("frame"), // [Deprecated] + []byte("rules"), // [Deprecated] + []byte("summary"), // [Deprecated] + []byte("width"), // [Deprecated] +) + +func (r *TableHTMLRenderer) renderTable(w util.BufWriter, source []byte, n gast.Node, entering bool) (gast.WalkStatus, error) { + if entering { + _, _ = w.WriteString("<table") + if n.Attributes() != nil { + html.RenderAttributes(w, n, TableAttributeFilter) + } + _, _ = w.WriteString(">\n") + } else { + _, _ = w.WriteString("</table>\n") + } + return gast.WalkContinue, nil +} + +// TableHeaderAttributeFilter defines attribute names which <thead> elements can have. +var TableHeaderAttributeFilter = html.GlobalAttributeFilter.Extend( + []byte("align"), // [Deprecated since HTML4] [Obsolete since HTML5] + []byte("bgcolor"), // [Not Standardized] + []byte("char"), // [Deprecated since HTML4] [Obsolete since HTML5] + []byte("charoff"), // [Deprecated since HTML4] [Obsolete since HTML5] + []byte("valign"), // [Deprecated since HTML4] [Obsolete since HTML5] +) + +func (r *TableHTMLRenderer) renderTableHeader(w util.BufWriter, source []byte, n gast.Node, entering bool) (gast.WalkStatus, error) { + if entering { + _, _ = w.WriteString("<thead") + if n.Attributes() != nil { + html.RenderAttributes(w, n, TableHeaderAttributeFilter) + } + _, _ = w.WriteString(">\n") + _, _ = w.WriteString("<tr>\n") // Header <tr> has no separate handle + } else { + _, _ = w.WriteString("</tr>\n") + _, _ = w.WriteString("</thead>\n") + if n.NextSibling() != nil { + _, _ = w.WriteString("<tbody>\n") + } + } + return gast.WalkContinue, nil +} + +// TableRowAttributeFilter defines attribute names which <tr> elements can have. +var TableRowAttributeFilter = html.GlobalAttributeFilter.Extend( + []byte("align"), // [Obsolete since HTML5] + []byte("bgcolor"), // [Obsolete since HTML5] + []byte("char"), // [Obsolete since HTML5] + []byte("charoff"), // [Obsolete since HTML5] + []byte("valign"), // [Obsolete since HTML5] +) + +func (r *TableHTMLRenderer) renderTableRow(w util.BufWriter, source []byte, n gast.Node, entering bool) (gast.WalkStatus, error) { + if entering { + _, _ = w.WriteString("<tr") + if n.Attributes() != nil { + html.RenderAttributes(w, n, TableRowAttributeFilter) + } + _, _ = w.WriteString(">\n") + } else { + _, _ = w.WriteString("</tr>\n") + if n.Parent().LastChild() == n { + _, _ = w.WriteString("</tbody>\n") + } + } + return gast.WalkContinue, nil +} + +// TableThCellAttributeFilter defines attribute names which table <th> cells can have. +var TableThCellAttributeFilter = html.GlobalAttributeFilter.Extend( + []byte("abbr"), // [OK] Contains a short abbreviated description of the cell's content [NOT OK in <td>] + + []byte("align"), // [Obsolete since HTML5] + []byte("axis"), // [Obsolete since HTML5] + []byte("bgcolor"), // [Not Standardized] + []byte("char"), // [Obsolete since HTML5] + []byte("charoff"), // [Obsolete since HTML5] + + []byte("colspan"), // [OK] Number of columns that the cell is to span + []byte("headers"), // [OK] This attribute contains a list of space-separated strings, each corresponding to the id attribute of the <th> elements that apply to this element + + []byte("height"), // [Deprecated since HTML4] [Obsolete since HTML5] + + []byte("rowspan"), // [OK] Number of rows that the cell is to span + []byte("scope"), // [OK] This enumerated attribute defines the cells that the header (defined in the <th>) element relates to [NOT OK in <td>] + + []byte("valign"), // [Obsolete since HTML5] + []byte("width"), // [Deprecated since HTML4] [Obsolete since HTML5] +) + +// TableTdCellAttributeFilter defines attribute names which table <td> cells can have. +var TableTdCellAttributeFilter = html.GlobalAttributeFilter.Extend( + []byte("abbr"), // [Obsolete since HTML5] [OK in <th>] + []byte("align"), // [Obsolete since HTML5] + []byte("axis"), // [Obsolete since HTML5] + []byte("bgcolor"), // [Not Standardized] + []byte("char"), // [Obsolete since HTML5] + []byte("charoff"), // [Obsolete since HTML5] + + []byte("colspan"), // [OK] Number of columns that the cell is to span + []byte("headers"), // [OK] This attribute contains a list of space-separated strings, each corresponding to the id attribute of the <th> elements that apply to this element + + []byte("height"), // [Deprecated since HTML4] [Obsolete since HTML5] + + []byte("rowspan"), // [OK] Number of rows that the cell is to span + + []byte("scope"), // [Obsolete since HTML5] [OK in <th>] + []byte("valign"), // [Obsolete since HTML5] + []byte("width"), // [Deprecated since HTML4] [Obsolete since HTML5] +) + +func (r *TableHTMLRenderer) renderTableCell(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) { + n := node.(*ast.TableCell) + tag := "td" + if n.Parent().Kind() == ast.KindTableHeader { + tag = "th" + } + if entering { + fmt.Fprintf(w, "<%s", tag) + if n.Alignment != ast.AlignNone { + amethod := r.TableConfig.TableCellAlignMethod + if amethod == TableCellAlignDefault { + if r.Config.XHTML { + amethod = TableCellAlignAttribute + } else { + amethod = TableCellAlignStyle + } + } + switch amethod { + case TableCellAlignAttribute: + if _, ok := n.AttributeString("align"); !ok { // Skip align render if overridden + fmt.Fprintf(w, ` align="%s"`, n.Alignment.String()) + } + case TableCellAlignStyle: + v, ok := n.AttributeString("style") + var cob util.CopyOnWriteBuffer + if ok { + cob = util.NewCopyOnWriteBuffer(v.([]byte)) + cob.AppendByte(';') + } + style := fmt.Sprintf("text-align:%s", n.Alignment.String()) + cob.AppendString(style) + n.SetAttributeString("style", cob.Bytes()) + } + } + if n.Attributes() != nil { + if tag == "td" { + html.RenderAttributes(w, n, TableTdCellAttributeFilter) // <td> + } else { + html.RenderAttributes(w, n, TableThCellAttributeFilter) // <th> + } + } + _ = w.WriteByte('>') + } else { + fmt.Fprintf(w, "</%s>\n", tag) + } + return gast.WalkContinue, nil +} + +type table struct { + options []TableOption +} + +// Table is an extension that allow you to use GFM tables . +var Table = &table{ + options: []TableOption{}, +} + +// NewTable returns a new extension with given options. +func NewTable(opts ...TableOption) goldmark.Extender { + return &table{ + options: opts, + } +} + +func (e *table) Extend(m goldmark.Markdown) { + m.Parser().AddOptions( + parser.WithParagraphTransformers( + util.Prioritized(NewTableParagraphTransformer(), 200), + ), + parser.WithASTTransformers( + util.Prioritized(defaultTableASTTransformer, 0), + ), + ) + m.Renderer().AddOptions(renderer.WithNodeRenderers( + util.Prioritized(NewTableHTMLRenderer(e.options...), 500), + )) +} diff --git a/vendor/github.com/yuin/goldmark/extension/tasklist.go b/vendor/github.com/yuin/goldmark/extension/tasklist.go new file mode 100644 index 00000000..1f3e52c2 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/extension/tasklist.go @@ -0,0 +1,115 @@ +package extension + +import ( + "github.com/yuin/goldmark" + gast "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" + "regexp" +) + +var taskListRegexp = regexp.MustCompile(`^\[([\sxX])\]\s*`) + +type taskCheckBoxParser struct { +} + +var defaultTaskCheckBoxParser = &taskCheckBoxParser{} + +// NewTaskCheckBoxParser returns a new InlineParser that can parse +// checkboxes in list items. +// This parser must take precedence over the parser.LinkParser. +func NewTaskCheckBoxParser() parser.InlineParser { + return defaultTaskCheckBoxParser +} + +func (s *taskCheckBoxParser) Trigger() []byte { + return []byte{'['} +} + +func (s *taskCheckBoxParser) Parse(parent gast.Node, block text.Reader, pc parser.Context) gast.Node { + // Given AST structure must be like + // - List + // - ListItem : parent.Parent + // - TextBlock : parent + // (current line) + if parent.Parent() == nil || parent.Parent().FirstChild() != parent { + return nil + } + + if _, ok := parent.Parent().(*gast.ListItem); !ok { + return nil + } + line, _ := block.PeekLine() + m := taskListRegexp.FindSubmatchIndex(line) + if m == nil { + return nil + } + value := line[m[2]:m[3]][0] + block.Advance(m[1]) + checked := value == 'x' || value == 'X' + return ast.NewTaskCheckBox(checked) +} + +func (s *taskCheckBoxParser) CloseBlock(parent gast.Node, pc parser.Context) { + // nothing to do +} + +// TaskCheckBoxHTMLRenderer is a renderer.NodeRenderer implementation that +// renders checkboxes in list items. +type TaskCheckBoxHTMLRenderer struct { + html.Config +} + +// NewTaskCheckBoxHTMLRenderer returns a new TaskCheckBoxHTMLRenderer. +func NewTaskCheckBoxHTMLRenderer(opts ...html.Option) renderer.NodeRenderer { + r := &TaskCheckBoxHTMLRenderer{ + Config: html.NewConfig(), + } + for _, opt := range opts { + opt.SetHTMLOption(&r.Config) + } + return r +} + +// RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs. +func (r *TaskCheckBoxHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + reg.Register(ast.KindTaskCheckBox, r.renderTaskCheckBox) +} + +func (r *TaskCheckBoxHTMLRenderer) renderTaskCheckBox(w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) { + if !entering { + return gast.WalkContinue, nil + } + n := node.(*ast.TaskCheckBox) + + if n.IsChecked { + w.WriteString(`<input checked="" disabled="" type="checkbox"`) + } else { + w.WriteString(`<input disabled="" type="checkbox"`) + } + if r.XHTML { + w.WriteString(" /> ") + } else { + w.WriteString("> ") + } + return gast.WalkContinue, nil +} + +type taskList struct { +} + +// TaskList is an extension that allow you to use GFM task lists. +var TaskList = &taskList{} + +func (e *taskList) Extend(m goldmark.Markdown) { + m.Parser().AddOptions(parser.WithInlineParsers( + util.Prioritized(NewTaskCheckBoxParser(), 0), + )) + m.Renderer().AddOptions(renderer.WithNodeRenderers( + util.Prioritized(NewTaskCheckBoxHTMLRenderer(), 500), + )) +} diff --git a/vendor/github.com/yuin/goldmark/extension/typographer.go b/vendor/github.com/yuin/goldmark/extension/typographer.go new file mode 100644 index 00000000..2c347309 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/extension/typographer.go @@ -0,0 +1,323 @@ +package extension + +import ( + "unicode" + + "github.com/yuin/goldmark" + gast "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +var uncloseCounterKey = parser.NewContextKey() + +type unclosedCounter struct { + Single int + Double int +} + +func (u *unclosedCounter) Reset() { + u.Single = 0 + u.Double = 0 +} + +func getUnclosedCounter(pc parser.Context) *unclosedCounter { + v := pc.Get(uncloseCounterKey) + if v == nil { + v = &unclosedCounter{} + pc.Set(uncloseCounterKey, v) + } + return v.(*unclosedCounter) +} + +// TypographicPunctuation is a key of the punctuations that can be replaced with +// typographic entities. +type TypographicPunctuation int + +const ( + // LeftSingleQuote is ' + LeftSingleQuote TypographicPunctuation = iota + 1 + // RightSingleQuote is ' + RightSingleQuote + // LeftDoubleQuote is " + LeftDoubleQuote + // RightDoubleQuote is " + RightDoubleQuote + // EnDash is -- + EnDash + // EmDash is --- + EmDash + // Ellipsis is ... + Ellipsis + // LeftAngleQuote is << + LeftAngleQuote + // RightAngleQuote is >> + RightAngleQuote + // Apostrophe is ' + Apostrophe + + typographicPunctuationMax +) + +// An TypographerConfig struct is a data structure that holds configuration of the +// Typographer extension. +type TypographerConfig struct { + Substitutions [][]byte +} + +func newDefaultSubstitutions() [][]byte { + replacements := make([][]byte, typographicPunctuationMax) + replacements[LeftSingleQuote] = []byte("‘") + replacements[RightSingleQuote] = []byte("’") + replacements[LeftDoubleQuote] = []byte("“") + replacements[RightDoubleQuote] = []byte("”") + replacements[EnDash] = []byte("–") + replacements[EmDash] = []byte("—") + replacements[Ellipsis] = []byte("…") + replacements[LeftAngleQuote] = []byte("«") + replacements[RightAngleQuote] = []byte("»") + replacements[Apostrophe] = []byte("’") + + return replacements +} + +// SetOption implements SetOptioner. +func (b *TypographerConfig) SetOption(name parser.OptionName, value interface{}) { + switch name { + case optTypographicSubstitutions: + b.Substitutions = value.([][]byte) + } +} + +// A TypographerOption interface sets options for the TypographerParser. +type TypographerOption interface { + parser.Option + SetTypographerOption(*TypographerConfig) +} + +const optTypographicSubstitutions parser.OptionName = "TypographicSubstitutions" + +// TypographicSubstitutions is a list of the substitutions for the Typographer extension. +type TypographicSubstitutions map[TypographicPunctuation][]byte + +type withTypographicSubstitutions struct { + value [][]byte +} + +func (o *withTypographicSubstitutions) SetParserOption(c *parser.Config) { + c.Options[optTypographicSubstitutions] = o.value +} + +func (o *withTypographicSubstitutions) SetTypographerOption(p *TypographerConfig) { + p.Substitutions = o.value +} + +// WithTypographicSubstitutions is a functional otpion that specify replacement text +// for punctuations. +func WithTypographicSubstitutions(values map[TypographicPunctuation][]byte) TypographerOption { + replacements := newDefaultSubstitutions() + for k, v := range values { + replacements[k] = v + } + + return &withTypographicSubstitutions{replacements} +} + +type typographerDelimiterProcessor struct { +} + +func (p *typographerDelimiterProcessor) IsDelimiter(b byte) bool { + return b == '\'' || b == '"' +} + +func (p *typographerDelimiterProcessor) CanOpenCloser(opener, closer *parser.Delimiter) bool { + return opener.Char == closer.Char +} + +func (p *typographerDelimiterProcessor) OnMatch(consumes int) gast.Node { + return nil +} + +var defaultTypographerDelimiterProcessor = &typographerDelimiterProcessor{} + +type typographerParser struct { + TypographerConfig +} + +// NewTypographerParser return a new InlineParser that parses +// typographer expressions. +func NewTypographerParser(opts ...TypographerOption) parser.InlineParser { + p := &typographerParser{ + TypographerConfig: TypographerConfig{ + Substitutions: newDefaultSubstitutions(), + }, + } + for _, o := range opts { + o.SetTypographerOption(&p.TypographerConfig) + } + return p +} + +func (s *typographerParser) Trigger() []byte { + return []byte{'\'', '"', '-', '.', '<', '>'} +} + +func (s *typographerParser) Parse(parent gast.Node, block text.Reader, pc parser.Context) gast.Node { + line, _ := block.PeekLine() + c := line[0] + if len(line) > 2 { + if c == '-' { + if s.Substitutions[EmDash] != nil && line[1] == '-' && line[2] == '-' { // --- + node := gast.NewString(s.Substitutions[EmDash]) + node.SetCode(true) + block.Advance(3) + return node + } + } else if c == '.' { + if s.Substitutions[Ellipsis] != nil && line[1] == '.' && line[2] == '.' { // ... + node := gast.NewString(s.Substitutions[Ellipsis]) + node.SetCode(true) + block.Advance(3) + return node + } + return nil + } + } + if len(line) > 1 { + if c == '<' { + if s.Substitutions[LeftAngleQuote] != nil && line[1] == '<' { // << + node := gast.NewString(s.Substitutions[LeftAngleQuote]) + node.SetCode(true) + block.Advance(2) + return node + } + return nil + } else if c == '>' { + if s.Substitutions[RightAngleQuote] != nil && line[1] == '>' { // >> + node := gast.NewString(s.Substitutions[RightAngleQuote]) + node.SetCode(true) + block.Advance(2) + return node + } + return nil + } else if s.Substitutions[EnDash] != nil && c == '-' && line[1] == '-' { // -- + node := gast.NewString(s.Substitutions[EnDash]) + node.SetCode(true) + block.Advance(2) + return node + } + } + if c == '\'' || c == '"' { + before := block.PrecendingCharacter() + d := parser.ScanDelimiter(line, before, 1, defaultTypographerDelimiterProcessor) + if d == nil { + return nil + } + counter := getUnclosedCounter(pc) + if c == '\'' { + if s.Substitutions[Apostrophe] != nil { + // Handle decade abbrevations such as '90s + if d.CanOpen && !d.CanClose && len(line) > 3 && util.IsNumeric(line[1]) && util.IsNumeric(line[2]) && line[3] == 's' { + after := rune(' ') + if len(line) > 4 { + after = util.ToRune(line, 4) + } + if len(line) == 3 || util.IsSpaceRune(after) || util.IsPunctRune(after) { + node := gast.NewString(s.Substitutions[Apostrophe]) + node.SetCode(true) + block.Advance(1) + return node + } + } + // Convert normal apostrophes. This is probably more flexible than necessary but + // converts any apostrophe in between two alphanumerics. + if len(line) > 1 && (unicode.IsDigit(before) || unicode.IsLetter(before)) && (unicode.IsLetter(util.ToRune(line, 1))) { + node := gast.NewString(s.Substitutions[Apostrophe]) + node.SetCode(true) + block.Advance(1) + return node + } + } + if s.Substitutions[LeftSingleQuote] != nil && d.CanOpen && !d.CanClose { + nt := LeftSingleQuote + // special cases: Alice's, I'm ,Don't, You'd + if len(line) > 1 && (line[1] == 's' || line[1] == 'm' || line[1] == 't' || line[1] == 'd') && (len(line) < 3 || util.IsPunct(line[2]) || util.IsSpace(line[2])) { + nt = RightSingleQuote + } + // special cases: I've, I'll, You're + if len(line) > 2 && ((line[1] == 'v' && line[2] == 'e') || (line[1] == 'l' && line[2] == 'l') || (line[1] == 'r' && line[2] == 'e')) && (len(line) < 4 || util.IsPunct(line[3]) || util.IsSpace(line[3])) { + nt = RightSingleQuote + } + if nt == LeftSingleQuote { + counter.Single++ + } + + node := gast.NewString(s.Substitutions[nt]) + node.SetCode(true) + block.Advance(1) + return node + } + if s.Substitutions[RightSingleQuote] != nil && counter.Single > 0 { + isClose := d.CanClose && !d.CanOpen + maybeClose := d.CanClose && d.CanOpen && len(line) > 1 && (line[1] == ',' || line[1] == '.' || line[1] == '!' || line[1] == '?') && (len(line) == 2 || (len(line) > 2 && util.IsPunct(line[2]) || util.IsSpace(line[2]))) + if isClose || maybeClose { + node := gast.NewString(s.Substitutions[RightSingleQuote]) + node.SetCode(true) + block.Advance(1) + counter.Single-- + return node + } + } + } + if c == '"' { + if s.Substitutions[LeftDoubleQuote] != nil && d.CanOpen && !d.CanClose { + node := gast.NewString(s.Substitutions[LeftDoubleQuote]) + node.SetCode(true) + block.Advance(1) + counter.Double++ + return node + } + if s.Substitutions[RightDoubleQuote] != nil && counter.Double > 0 { + isClose := d.CanClose && !d.CanOpen + maybeClose := d.CanClose && d.CanOpen && len(line) > 1 && (line[1] == ',' || line[1] == '.' || line[1] == '!' || line[1] == '?') && (len(line) == 2 || (len(line) > 2 && util.IsPunct(line[2]) || util.IsSpace(line[2]))) + if isClose || maybeClose { + // special case: "Monitor 21"" + if len(line) > 1 && line[1] == '"' && unicode.IsDigit(before) { + return nil + } + node := gast.NewString(s.Substitutions[RightDoubleQuote]) + node.SetCode(true) + block.Advance(1) + counter.Double-- + return node + } + } + } + } + return nil +} + +func (s *typographerParser) CloseBlock(parent gast.Node, pc parser.Context) { + getUnclosedCounter(pc).Reset() +} + +type typographer struct { + options []TypographerOption +} + +// Typographer is an extension that replaces punctuations with typographic entities. +var Typographer = &typographer{} + +// NewTypographer returns a new Extender that replaces punctuations with typographic entities. +func NewTypographer(opts ...TypographerOption) goldmark.Extender { + return &typographer{ + options: opts, + } +} + +func (e *typographer) Extend(m goldmark.Markdown) { + m.Parser().AddOptions(parser.WithInlineParsers( + util.Prioritized(NewTypographerParser(e.options...), 9999), + )) +} diff --git a/vendor/github.com/yuin/goldmark/markdown.go b/vendor/github.com/yuin/goldmark/markdown.go new file mode 100644 index 00000000..86d12e22 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/markdown.go @@ -0,0 +1,140 @@ +// Package goldmark implements functions to convert markdown text to a desired format. +package goldmark + +import ( + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" + "io" +) + +// DefaultParser returns a new Parser that is configured by default values. +func DefaultParser() parser.Parser { + return parser.NewParser(parser.WithBlockParsers(parser.DefaultBlockParsers()...), + parser.WithInlineParsers(parser.DefaultInlineParsers()...), + parser.WithParagraphTransformers(parser.DefaultParagraphTransformers()...), + ) +} + +// DefaultRenderer returns a new Renderer that is configured by default values. +func DefaultRenderer() renderer.Renderer { + return renderer.NewRenderer(renderer.WithNodeRenderers(util.Prioritized(html.NewRenderer(), 1000))) +} + +var defaultMarkdown = New() + +// Convert interprets a UTF-8 bytes source in Markdown and +// write rendered contents to a writer w. +func Convert(source []byte, w io.Writer, opts ...parser.ParseOption) error { + return defaultMarkdown.Convert(source, w, opts...) +} + +// A Markdown interface offers functions to convert Markdown text to +// a desired format. +type Markdown interface { + // Convert interprets a UTF-8 bytes source in Markdown and write rendered + // contents to a writer w. + Convert(source []byte, writer io.Writer, opts ...parser.ParseOption) error + + // Parser returns a Parser that will be used for conversion. + Parser() parser.Parser + + // SetParser sets a Parser to this object. + SetParser(parser.Parser) + + // Parser returns a Renderer that will be used for conversion. + Renderer() renderer.Renderer + + // SetRenderer sets a Renderer to this object. + SetRenderer(renderer.Renderer) +} + +// Option is a functional option type for Markdown objects. +type Option func(*markdown) + +// WithExtensions adds extensions. +func WithExtensions(ext ...Extender) Option { + return func(m *markdown) { + m.extensions = append(m.extensions, ext...) + } +} + +// WithParser allows you to override the default parser. +func WithParser(p parser.Parser) Option { + return func(m *markdown) { + m.parser = p + } +} + +// WithParserOptions applies options for the parser. +func WithParserOptions(opts ...parser.Option) Option { + return func(m *markdown) { + m.parser.AddOptions(opts...) + } +} + +// WithRenderer allows you to override the default renderer. +func WithRenderer(r renderer.Renderer) Option { + return func(m *markdown) { + m.renderer = r + } +} + +// WithRendererOptions applies options for the renderer. +func WithRendererOptions(opts ...renderer.Option) Option { + return func(m *markdown) { + m.renderer.AddOptions(opts...) + } +} + +type markdown struct { + parser parser.Parser + renderer renderer.Renderer + extensions []Extender +} + +// New returns a new Markdown with given options. +func New(options ...Option) Markdown { + md := &markdown{ + parser: DefaultParser(), + renderer: DefaultRenderer(), + extensions: []Extender{}, + } + for _, opt := range options { + opt(md) + } + for _, e := range md.extensions { + e.Extend(md) + } + return md +} + +func (m *markdown) Convert(source []byte, writer io.Writer, opts ...parser.ParseOption) error { + reader := text.NewReader(source) + doc := m.parser.Parse(reader, opts...) + return m.renderer.Render(writer, source, doc) +} + +func (m *markdown) Parser() parser.Parser { + return m.parser +} + +func (m *markdown) SetParser(v parser.Parser) { + m.parser = v +} + +func (m *markdown) Renderer() renderer.Renderer { + return m.renderer +} + +func (m *markdown) SetRenderer(v renderer.Renderer) { + m.renderer = v +} + +// An Extender interface is used for extending Markdown. +type Extender interface { + // Extend extends the Markdown. + Extend(Markdown) +} diff --git a/vendor/github.com/yuin/goldmark/parser/attribute.go b/vendor/github.com/yuin/goldmark/parser/attribute.go new file mode 100644 index 00000000..f86c8361 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/attribute.go @@ -0,0 +1,328 @@ +package parser + +import ( + "bytes" + "io" + "strconv" + + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +var attrNameID = []byte("id") +var attrNameClass = []byte("class") + +// An Attribute is an attribute of the markdown elements +type Attribute struct { + Name []byte + Value interface{} +} + +// An Attributes is a collection of attributes. +type Attributes []Attribute + +// Find returns a (value, true) if an attribute correspond with given name is found, otherwise (nil, false). +func (as Attributes) Find(name []byte) (interface{}, bool) { + for _, a := range as { + if bytes.Equal(a.Name, name) { + return a.Value, true + } + } + return nil, false +} + +func (as Attributes) findUpdate(name []byte, cb func(v interface{}) interface{}) bool { + for i, a := range as { + if bytes.Equal(a.Name, name) { + as[i].Value = cb(a.Value) + return true + } + } + return false +} + +// ParseAttributes parses attributes into a map. +// ParseAttributes returns a parsed attributes and true if could parse +// attributes, otherwise nil and false. +func ParseAttributes(reader text.Reader) (Attributes, bool) { + savedLine, savedPosition := reader.Position() + reader.SkipSpaces() + if reader.Peek() != '{' { + reader.SetPosition(savedLine, savedPosition) + return nil, false + } + reader.Advance(1) + attrs := Attributes{} + for { + if reader.Peek() == '}' { + reader.Advance(1) + return attrs, true + } + attr, ok := parseAttribute(reader) + if !ok { + reader.SetPosition(savedLine, savedPosition) + return nil, false + } + if bytes.Equal(attr.Name, attrNameClass) { + if !attrs.findUpdate(attrNameClass, func(v interface{}) interface{} { + ret := make([]byte, 0, len(v.([]byte))+1+len(attr.Value.([]byte))) + ret = append(ret, v.([]byte)...) + return append(append(ret, ' '), attr.Value.([]byte)...) + }) { + attrs = append(attrs, attr) + } + } else { + attrs = append(attrs, attr) + } + reader.SkipSpaces() + if reader.Peek() == ',' { + reader.Advance(1) + reader.SkipSpaces() + } + } +} + +func parseAttribute(reader text.Reader) (Attribute, bool) { + reader.SkipSpaces() + c := reader.Peek() + if c == '#' || c == '.' { + reader.Advance(1) + line, _ := reader.PeekLine() + i := 0 + // HTML5 allows any kind of characters as id, but XHTML restricts characters for id. + // CommonMark is basically defined for XHTML(even though it is legacy). + // So we restrict id characters. + for ; i < len(line) && !util.IsSpace(line[i]) && + (!util.IsPunct(line[i]) || line[i] == '_' || line[i] == '-' || line[i] == ':' || line[i] == '.'); i++ { + } + name := attrNameClass + if c == '#' { + name = attrNameID + } + reader.Advance(i) + return Attribute{Name: name, Value: line[0:i]}, true + } + line, _ := reader.PeekLine() + if len(line) == 0 { + return Attribute{}, false + } + c = line[0] + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + c == '_' || c == ':') { + return Attribute{}, false + } + i := 0 + for ; i < len(line); i++ { + c = line[i] + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '_' || c == ':' || c == '.' || c == '-') { + break + } + } + name := line[:i] + reader.Advance(i) + reader.SkipSpaces() + c = reader.Peek() + if c != '=' { + return Attribute{}, false + } + reader.Advance(1) + reader.SkipSpaces() + value, ok := parseAttributeValue(reader) + if !ok { + return Attribute{}, false + } + if bytes.Equal(name, attrNameClass) { + if _, ok = value.([]byte); !ok { + return Attribute{}, false + } + } + return Attribute{Name: name, Value: value}, true +} + +func parseAttributeValue(reader text.Reader) (interface{}, bool) { + reader.SkipSpaces() + c := reader.Peek() + var value interface{} + ok := false + switch c { + case text.EOF: + return Attribute{}, false + case '{': + value, ok = ParseAttributes(reader) + case '[': + value, ok = parseAttributeArray(reader) + case '"': + value, ok = parseAttributeString(reader) + default: + if c == '-' || c == '+' || util.IsNumeric(c) { + value, ok = parseAttributeNumber(reader) + } else { + value, ok = parseAttributeOthers(reader) + } + } + if !ok { + return nil, false + } + return value, true +} + +func parseAttributeArray(reader text.Reader) ([]interface{}, bool) { + reader.Advance(1) // skip [ + ret := []interface{}{} + for i := 0; ; i++ { + c := reader.Peek() + comma := false + if i != 0 && c == ',' { + reader.Advance(1) + comma = true + } + if c == ']' { + if !comma { + reader.Advance(1) + return ret, true + } + return nil, false + } + reader.SkipSpaces() + value, ok := parseAttributeValue(reader) + if !ok { + return nil, false + } + ret = append(ret, value) + reader.SkipSpaces() + } +} + +func parseAttributeString(reader text.Reader) ([]byte, bool) { + reader.Advance(1) // skip " + line, _ := reader.PeekLine() + i := 0 + l := len(line) + var buf bytes.Buffer + for i < l { + c := line[i] + if c == '\\' && i != l-1 { + n := line[i+1] + switch n { + case '"', '/', '\\': + buf.WriteByte(n) + i += 2 + case 'b': + buf.WriteString("\b") + i += 2 + case 'f': + buf.WriteString("\f") + i += 2 + case 'n': + buf.WriteString("\n") + i += 2 + case 'r': + buf.WriteString("\r") + i += 2 + case 't': + buf.WriteString("\t") + i += 2 + default: + buf.WriteByte('\\') + i++ + } + continue + } + if c == '"' { + reader.Advance(i + 1) + return buf.Bytes(), true + } + buf.WriteByte(c) + i++ + } + return nil, false +} + +func scanAttributeDecimal(reader text.Reader, w io.ByteWriter) { + for { + c := reader.Peek() + if util.IsNumeric(c) { + w.WriteByte(c) + } else { + return + } + reader.Advance(1) + } +} + +func parseAttributeNumber(reader text.Reader) (float64, bool) { + sign := 1 + c := reader.Peek() + if c == '-' { + sign = -1 + reader.Advance(1) + } else if c == '+' { + reader.Advance(1) + } + var buf bytes.Buffer + if !util.IsNumeric(reader.Peek()) { + return 0, false + } + scanAttributeDecimal(reader, &buf) + if buf.Len() == 0 { + return 0, false + } + c = reader.Peek() + if c == '.' { + buf.WriteByte(c) + reader.Advance(1) + scanAttributeDecimal(reader, &buf) + } + c = reader.Peek() + if c == 'e' || c == 'E' { + buf.WriteByte(c) + reader.Advance(1) + c = reader.Peek() + if c == '-' || c == '+' { + buf.WriteByte(c) + reader.Advance(1) + } + scanAttributeDecimal(reader, &buf) + } + f, err := strconv.ParseFloat(buf.String(), 10) + if err != nil { + return 0, false + } + return float64(sign) * f, true +} + +var bytesTrue = []byte("true") +var bytesFalse = []byte("false") +var bytesNull = []byte("null") + +func parseAttributeOthers(reader text.Reader) (interface{}, bool) { + line, _ := reader.PeekLine() + c := line[0] + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + c == '_' || c == ':') { + return nil, false + } + i := 0 + for ; i < len(line); i++ { + c := line[i] + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '_' || c == ':' || c == '.' || c == '-') { + break + } + } + value := line[:i] + reader.Advance(i) + if bytes.Equal(value, bytesTrue) { + return true, true + } + if bytes.Equal(value, bytesFalse) { + return false, true + } + if bytes.Equal(value, bytesNull) { + return nil, true + } + return value, true +} diff --git a/vendor/github.com/yuin/goldmark/parser/atx_heading.go b/vendor/github.com/yuin/goldmark/parser/atx_heading.go new file mode 100644 index 00000000..13a198b5 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/atx_heading.go @@ -0,0 +1,246 @@ +package parser + +import ( + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +// A HeadingConfig struct is a data structure that holds configuration of the renderers related to headings. +type HeadingConfig struct { + AutoHeadingID bool + Attribute bool +} + +// SetOption implements SetOptioner. +func (b *HeadingConfig) SetOption(name OptionName, value interface{}) { + switch name { + case optAutoHeadingID: + b.AutoHeadingID = true + case optAttribute: + b.Attribute = true + } +} + +// A HeadingOption interface sets options for heading parsers. +type HeadingOption interface { + Option + SetHeadingOption(*HeadingConfig) +} + +// AutoHeadingID is an option name that enables auto IDs for headings. +const optAutoHeadingID OptionName = "AutoHeadingID" + +type withAutoHeadingID struct { +} + +func (o *withAutoHeadingID) SetParserOption(c *Config) { + c.Options[optAutoHeadingID] = true +} + +func (o *withAutoHeadingID) SetHeadingOption(p *HeadingConfig) { + p.AutoHeadingID = true +} + +// WithAutoHeadingID is a functional option that enables custom heading ids and +// auto generated heading ids. +func WithAutoHeadingID() HeadingOption { + return &withAutoHeadingID{} +} + +type withHeadingAttribute struct { + Option +} + +func (o *withHeadingAttribute) SetHeadingOption(p *HeadingConfig) { + p.Attribute = true +} + +// WithHeadingAttribute is a functional option that enables custom heading attributes. +func WithHeadingAttribute() HeadingOption { + return &withHeadingAttribute{WithAttribute()} +} + +type atxHeadingParser struct { + HeadingConfig +} + +// NewATXHeadingParser return a new BlockParser that can parse ATX headings. +func NewATXHeadingParser(opts ...HeadingOption) BlockParser { + p := &atxHeadingParser{} + for _, o := range opts { + o.SetHeadingOption(&p.HeadingConfig) + } + return p +} + +func (b *atxHeadingParser) Trigger() []byte { + return []byte{'#'} +} + +func (b *atxHeadingParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) { + line, segment := reader.PeekLine() + pos := pc.BlockOffset() + if pos < 0 { + return nil, NoChildren + } + i := pos + for ; i < len(line) && line[i] == '#'; i++ { + } + level := i - pos + if i == pos || level > 6 { + return nil, NoChildren + } + if i == len(line) { // alone '#' (without a new line character) + return ast.NewHeading(level), NoChildren + } + l := util.TrimLeftSpaceLength(line[i:]) + if l == 0 { + return nil, NoChildren + } + start := i + l + if start >= len(line) { + start = len(line) - 1 + } + origstart := start + stop := len(line) - util.TrimRightSpaceLength(line) + + node := ast.NewHeading(level) + parsed := false + if b.Attribute { // handles special case like ### heading ### {#id} + start-- + closureClose := -1 + closureOpen := -1 + for j := start; j < stop; { + c := line[j] + if util.IsEscapedPunctuation(line, j) { + j += 2 + } else if util.IsSpace(c) && j < stop-1 && line[j+1] == '#' { + closureOpen = j + 1 + k := j + 1 + for ; k < stop && line[k] == '#'; k++ { + } + closureClose = k + break + } else { + j++ + } + } + if closureClose > 0 { + reader.Advance(closureClose) + attrs, ok := ParseAttributes(reader) + rest, _ := reader.PeekLine() + parsed = ok && util.IsBlank(rest) + if parsed { + for _, attr := range attrs { + node.SetAttribute(attr.Name, attr.Value) + } + node.Lines().Append(text.NewSegment(segment.Start+start+1-segment.Padding, segment.Start+closureOpen-segment.Padding)) + } + } + } + if !parsed { + start = origstart + stop := len(line) - util.TrimRightSpaceLength(line) + if stop <= start { // empty headings like '##[space]' + stop = start + } else { + i = stop - 1 + for ; line[i] == '#' && i >= start; i-- { + } + if i != stop-1 && !util.IsSpace(line[i]) { + i = stop - 1 + } + i++ + stop = i + } + + if len(util.TrimRight(line[start:stop], []byte{'#'})) != 0 { // empty heading like '### ###' + node.Lines().Append(text.NewSegment(segment.Start+start-segment.Padding, segment.Start+stop-segment.Padding)) + } + } + return node, NoChildren +} + +func (b *atxHeadingParser) Continue(node ast.Node, reader text.Reader, pc Context) State { + return Close +} + +func (b *atxHeadingParser) Close(node ast.Node, reader text.Reader, pc Context) { + if b.Attribute { + _, ok := node.AttributeString("id") + if !ok { + parseLastLineAttributes(node, reader, pc) + } + } + + if b.AutoHeadingID { + id, ok := node.AttributeString("id") + if !ok { + generateAutoHeadingID(node.(*ast.Heading), reader, pc) + } else { + pc.IDs().Put(id.([]byte)) + } + } +} + +func (b *atxHeadingParser) CanInterruptParagraph() bool { + return true +} + +func (b *atxHeadingParser) CanAcceptIndentedLine() bool { + return false +} + +func generateAutoHeadingID(node *ast.Heading, reader text.Reader, pc Context) { + var line []byte + lastIndex := node.Lines().Len() - 1 + if lastIndex > -1 { + lastLine := node.Lines().At(lastIndex) + line = lastLine.Value(reader.Source()) + } + headingID := pc.IDs().Generate(line, ast.KindHeading) + node.SetAttribute(attrNameID, headingID) +} + +func parseLastLineAttributes(node ast.Node, reader text.Reader, pc Context) { + lastIndex := node.Lines().Len() - 1 + if lastIndex < 0 { // empty headings + return + } + lastLine := node.Lines().At(lastIndex) + line := lastLine.Value(reader.Source()) + lr := text.NewReader(line) + var attrs Attributes + var ok bool + var start text.Segment + var sl int + var end text.Segment + for { + c := lr.Peek() + if c == text.EOF { + break + } + if c == '\\' { + lr.Advance(1) + if lr.Peek() == '{' { + lr.Advance(1) + } + continue + } + if c == '{' { + sl, start = lr.Position() + attrs, ok = ParseAttributes(lr) + _, end = lr.Position() + lr.SetPosition(sl, start) + } + lr.Advance(1) + } + if ok && util.IsBlank(line[end.Start:]) { + for _, attr := range attrs { + node.SetAttribute(attr.Name, attr.Value) + } + lastLine.Stop = lastLine.Start + start.Start + node.Lines().Set(lastIndex, lastLine) + } +} diff --git a/vendor/github.com/yuin/goldmark/parser/auto_link.go b/vendor/github.com/yuin/goldmark/parser/auto_link.go new file mode 100644 index 00000000..726a5057 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/auto_link.go @@ -0,0 +1,42 @@ +package parser + +import ( + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +type autoLinkParser struct { +} + +var defaultAutoLinkParser = &autoLinkParser{} + +// NewAutoLinkParser returns a new InlineParser that parses autolinks +// surrounded by '<' and '>' . +func NewAutoLinkParser() InlineParser { + return defaultAutoLinkParser +} + +func (s *autoLinkParser) Trigger() []byte { + return []byte{'<'} +} + +func (s *autoLinkParser) Parse(parent ast.Node, block text.Reader, pc Context) ast.Node { + line, segment := block.PeekLine() + stop := util.FindEmailIndex(line[1:]) + typ := ast.AutoLinkType(ast.AutoLinkEmail) + if stop < 0 { + stop = util.FindURLIndex(line[1:]) + typ = ast.AutoLinkURL + } + if stop < 0 { + return nil + } + stop++ + if stop >= len(line) || line[stop] != '>' { + return nil + } + value := ast.NewTextSegment(text.NewSegment(segment.Start+1, segment.Start+stop)) + block.Advance(stop + 1) + return ast.NewAutoLink(typ, value) +} diff --git a/vendor/github.com/yuin/goldmark/parser/blockquote.go b/vendor/github.com/yuin/goldmark/parser/blockquote.go new file mode 100644 index 00000000..e7778dca --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/blockquote.go @@ -0,0 +1,69 @@ +package parser + +import ( + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +type blockquoteParser struct { +} + +var defaultBlockquoteParser = &blockquoteParser{} + +// NewBlockquoteParser returns a new BlockParser that +// parses blockquotes. +func NewBlockquoteParser() BlockParser { + return defaultBlockquoteParser +} + +func (b *blockquoteParser) process(reader text.Reader) bool { + line, _ := reader.PeekLine() + w, pos := util.IndentWidth(line, reader.LineOffset()) + if w > 3 || pos >= len(line) || line[pos] != '>' { + return false + } + pos++ + if pos >= len(line) || line[pos] == '\n' { + reader.Advance(pos) + return true + } + if line[pos] == ' ' || line[pos] == '\t' { + pos++ + } + reader.Advance(pos) + if line[pos-1] == '\t' { + reader.SetPadding(2) + } + return true +} + +func (b *blockquoteParser) Trigger() []byte { + return []byte{'>'} +} + +func (b *blockquoteParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) { + if b.process(reader) { + return ast.NewBlockquote(), HasChildren + } + return nil, NoChildren +} + +func (b *blockquoteParser) Continue(node ast.Node, reader text.Reader, pc Context) State { + if b.process(reader) { + return Continue | HasChildren + } + return Close +} + +func (b *blockquoteParser) Close(node ast.Node, reader text.Reader, pc Context) { + // nothing to do +} + +func (b *blockquoteParser) CanInterruptParagraph() bool { + return true +} + +func (b *blockquoteParser) CanAcceptIndentedLine() bool { + return false +} diff --git a/vendor/github.com/yuin/goldmark/parser/code_block.go b/vendor/github.com/yuin/goldmark/parser/code_block.go new file mode 100644 index 00000000..732f18c6 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/code_block.go @@ -0,0 +1,100 @@ +package parser + +import ( + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +type codeBlockParser struct { +} + +// CodeBlockParser is a BlockParser implementation that parses indented code blocks. +var defaultCodeBlockParser = &codeBlockParser{} + +// NewCodeBlockParser returns a new BlockParser that +// parses code blocks. +func NewCodeBlockParser() BlockParser { + return defaultCodeBlockParser +} + +func (b *codeBlockParser) Trigger() []byte { + return nil +} + +func (b *codeBlockParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) { + line, segment := reader.PeekLine() + pos, padding := util.IndentPosition(line, reader.LineOffset(), 4) + if pos < 0 || util.IsBlank(line) { + return nil, NoChildren + } + node := ast.NewCodeBlock() + reader.AdvanceAndSetPadding(pos, padding) + _, segment = reader.PeekLine() + // if code block line starts with a tab, keep a tab as it is. + if segment.Padding != 0 { + preserveLeadingTabInCodeBlock(&segment, reader, 0) + } + node.Lines().Append(segment) + reader.Advance(segment.Len() - 1) + return node, NoChildren + +} + +func (b *codeBlockParser) Continue(node ast.Node, reader text.Reader, pc Context) State { + line, segment := reader.PeekLine() + if util.IsBlank(line) { + node.Lines().Append(segment.TrimLeftSpaceWidth(4, reader.Source())) + return Continue | NoChildren + } + pos, padding := util.IndentPosition(line, reader.LineOffset(), 4) + if pos < 0 { + return Close + } + reader.AdvanceAndSetPadding(pos, padding) + _, segment = reader.PeekLine() + + // if code block line starts with a tab, keep a tab as it is. + if segment.Padding != 0 { + preserveLeadingTabInCodeBlock(&segment, reader, 0) + } + + node.Lines().Append(segment) + reader.Advance(segment.Len() - 1) + return Continue | NoChildren +} + +func (b *codeBlockParser) Close(node ast.Node, reader text.Reader, pc Context) { + // trim trailing blank lines + lines := node.Lines() + length := lines.Len() - 1 + source := reader.Source() + for length >= 0 { + line := lines.At(length) + if util.IsBlank(line.Value(source)) { + length-- + } else { + break + } + } + lines.SetSliced(0, length+1) +} + +func (b *codeBlockParser) CanInterruptParagraph() bool { + return false +} + +func (b *codeBlockParser) CanAcceptIndentedLine() bool { + return true +} + +func preserveLeadingTabInCodeBlock(segment *text.Segment, reader text.Reader, indent int) { + offsetWithPadding := reader.LineOffset() + indent + sl, ss := reader.Position() + reader.SetPosition(sl, text.NewSegment(ss.Start-1, ss.Stop)) + if offsetWithPadding == reader.LineOffset() { + segment.Padding = 0 + segment.Start-- + } + reader.SetPosition(sl, ss) +} diff --git a/vendor/github.com/yuin/goldmark/parser/code_span.go b/vendor/github.com/yuin/goldmark/parser/code_span.go new file mode 100644 index 00000000..a74b09bc --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/code_span.go @@ -0,0 +1,84 @@ +package parser + +import ( + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" +) + +type codeSpanParser struct { +} + +var defaultCodeSpanParser = &codeSpanParser{} + +// NewCodeSpanParser return a new InlineParser that parses inline codes +// surrounded by '`' . +func NewCodeSpanParser() InlineParser { + return defaultCodeSpanParser +} + +func (s *codeSpanParser) Trigger() []byte { + return []byte{'`'} +} + +func (s *codeSpanParser) Parse(parent ast.Node, block text.Reader, pc Context) ast.Node { + line, startSegment := block.PeekLine() + opener := 0 + for ; opener < len(line) && line[opener] == '`'; opener++ { + } + block.Advance(opener) + l, pos := block.Position() + node := ast.NewCodeSpan() + for { + line, segment := block.PeekLine() + if line == nil { + block.SetPosition(l, pos) + return ast.NewTextSegment(startSegment.WithStop(startSegment.Start + opener)) + } + for i := 0; i < len(line); i++ { + c := line[i] + if c == '`' { + oldi := i + for ; i < len(line) && line[i] == '`'; i++ { + } + closure := i - oldi + if closure == opener && (i >= len(line) || line[i] != '`') { + segment = segment.WithStop(segment.Start + i - closure) + if !segment.IsEmpty() { + node.AppendChild(node, ast.NewRawTextSegment(segment)) + } + block.Advance(i) + goto end + } + } + } + node.AppendChild(node, ast.NewRawTextSegment(segment)) + block.AdvanceLine() + } +end: + if !node.IsBlank(block.Source()) { + // trim first halfspace and last halfspace + segment := node.FirstChild().(*ast.Text).Segment + shouldTrimmed := true + if !(!segment.IsEmpty() && isSpaceOrNewline(block.Source()[segment.Start])) { + shouldTrimmed = false + } + segment = node.LastChild().(*ast.Text).Segment + if !(!segment.IsEmpty() && isSpaceOrNewline(block.Source()[segment.Stop-1])) { + shouldTrimmed = false + } + if shouldTrimmed { + t := node.FirstChild().(*ast.Text) + segment := t.Segment + t.Segment = segment.WithStart(segment.Start + 1) + t = node.LastChild().(*ast.Text) + segment = node.LastChild().(*ast.Text).Segment + t.Segment = segment.WithStop(segment.Stop - 1) + } + + } + return node +} + +func isSpaceOrNewline(c byte) bool { + return c == ' ' || c == '\n' +} diff --git a/vendor/github.com/yuin/goldmark/parser/delimiter.go b/vendor/github.com/yuin/goldmark/parser/delimiter.go new file mode 100644 index 00000000..eb843af4 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/delimiter.go @@ -0,0 +1,238 @@ +package parser + +import ( + "fmt" + "strings" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +// A DelimiterProcessor interface provides a set of functions about +// Delimiter nodes. +type DelimiterProcessor interface { + // IsDelimiter returns true if given character is a delimiter, otherwise false. + IsDelimiter(byte) bool + + // CanOpenCloser returns true if given opener can close given closer, otherwise false. + CanOpenCloser(opener, closer *Delimiter) bool + + // OnMatch will be called when new matched delimiter found. + // OnMatch should return a new Node correspond to the matched delimiter. + OnMatch(consumes int) ast.Node +} + +// A Delimiter struct represents a delimiter like '*' of the Markdown text. +type Delimiter struct { + ast.BaseInline + + Segment text.Segment + + // CanOpen is set true if this delimiter can open a span for a new node. + // See https://spec.commonmark.org/0.30/#can-open-emphasis for details. + CanOpen bool + + // CanClose is set true if this delimiter can close a span for a new node. + // See https://spec.commonmark.org/0.30/#can-open-emphasis for details. + CanClose bool + + // Length is a remaining length of this delimiter. + Length int + + // OriginalLength is a original length of this delimiter. + OriginalLength int + + // Char is a character of this delimiter. + Char byte + + // PreviousDelimiter is a previous sibling delimiter node of this delimiter. + PreviousDelimiter *Delimiter + + // NextDelimiter is a next sibling delimiter node of this delimiter. + NextDelimiter *Delimiter + + // Processor is a DelimiterProcessor associated with this delimiter. + Processor DelimiterProcessor +} + +// Inline implements Inline.Inline. +func (d *Delimiter) Inline() {} + +// Dump implements Node.Dump. +func (d *Delimiter) Dump(source []byte, level int) { + fmt.Printf("%sDelimiter: \"%s\"\n", strings.Repeat(" ", level), string(d.Text(source))) +} + +var kindDelimiter = ast.NewNodeKind("Delimiter") + +// Kind implements Node.Kind +func (d *Delimiter) Kind() ast.NodeKind { + return kindDelimiter +} + +// Text implements Node.Text +func (d *Delimiter) Text(source []byte) []byte { + return d.Segment.Value(source) +} + +// ConsumeCharacters consumes delimiters. +func (d *Delimiter) ConsumeCharacters(n int) { + d.Length -= n + d.Segment = d.Segment.WithStop(d.Segment.Start + d.Length) +} + +// CalcComsumption calculates how many characters should be used for opening +// a new span correspond to given closer. +func (d *Delimiter) CalcComsumption(closer *Delimiter) int { + if (d.CanClose || closer.CanOpen) && (d.OriginalLength+closer.OriginalLength)%3 == 0 && closer.OriginalLength%3 != 0 { + return 0 + } + if d.Length >= 2 && closer.Length >= 2 { + return 2 + } + return 1 +} + +// NewDelimiter returns a new Delimiter node. +func NewDelimiter(canOpen, canClose bool, length int, char byte, processor DelimiterProcessor) *Delimiter { + c := &Delimiter{ + BaseInline: ast.BaseInline{}, + CanOpen: canOpen, + CanClose: canClose, + Length: length, + OriginalLength: length, + Char: char, + PreviousDelimiter: nil, + NextDelimiter: nil, + Processor: processor, + } + return c +} + +// ScanDelimiter scans a delimiter by given DelimiterProcessor. +func ScanDelimiter(line []byte, before rune, min int, processor DelimiterProcessor) *Delimiter { + i := 0 + c := line[i] + j := i + if !processor.IsDelimiter(c) { + return nil + } + for ; j < len(line) && c == line[j]; j++ { + } + if (j - i) >= min { + after := rune(' ') + if j != len(line) { + after = util.ToRune(line, j) + } + + canOpen, canClose := false, false + beforeIsPunctuation := util.IsPunctRune(before) + beforeIsWhitespace := util.IsSpaceRune(before) + afterIsPunctuation := util.IsPunctRune(after) + afterIsWhitespace := util.IsSpaceRune(after) + + isLeft := !afterIsWhitespace && + (!afterIsPunctuation || beforeIsWhitespace || beforeIsPunctuation) + isRight := !beforeIsWhitespace && + (!beforeIsPunctuation || afterIsWhitespace || afterIsPunctuation) + + if line[i] == '_' { + canOpen = isLeft && (!isRight || beforeIsPunctuation) + canClose = isRight && (!isLeft || afterIsPunctuation) + } else { + canOpen = isLeft + canClose = isRight + } + return NewDelimiter(canOpen, canClose, j-i, c, processor) + } + return nil +} + +// ProcessDelimiters processes the delimiter list in the context. +// Processing will be stop when reaching the bottom. +// +// If you implement an inline parser that can have other inline nodes as +// children, you should call this function when nesting span has closed. +func ProcessDelimiters(bottom ast.Node, pc Context) { + lastDelimiter := pc.LastDelimiter() + if lastDelimiter == nil { + return + } + var closer *Delimiter + if bottom != nil { + if bottom != lastDelimiter { + for c := lastDelimiter.PreviousSibling(); c != nil && c != bottom; { + if d, ok := c.(*Delimiter); ok { + closer = d + } + c = c.PreviousSibling() + } + } + } else { + closer = pc.FirstDelimiter() + } + if closer == nil { + pc.ClearDelimiters(bottom) + return + } + for closer != nil { + if !closer.CanClose { + closer = closer.NextDelimiter + continue + } + consume := 0 + found := false + maybeOpener := false + var opener *Delimiter + for opener = closer.PreviousDelimiter; opener != nil && opener != bottom; opener = opener.PreviousDelimiter { + if opener.CanOpen && opener.Processor.CanOpenCloser(opener, closer) { + maybeOpener = true + consume = opener.CalcComsumption(closer) + if consume > 0 { + found = true + break + } + } + } + if !found { + next := closer.NextDelimiter + if !maybeOpener && !closer.CanOpen { + pc.RemoveDelimiter(closer) + } + closer = next + continue + } + opener.ConsumeCharacters(consume) + closer.ConsumeCharacters(consume) + + node := opener.Processor.OnMatch(consume) + + parent := opener.Parent() + child := opener.NextSibling() + + for child != nil && child != closer { + next := child.NextSibling() + node.AppendChild(node, child) + child = next + } + parent.InsertAfter(parent, opener, node) + + for c := opener.NextDelimiter; c != nil && c != closer; { + next := c.NextDelimiter + pc.RemoveDelimiter(c) + c = next + } + + if opener.Length == 0 { + pc.RemoveDelimiter(opener) + } + + if closer.Length == 0 { + next := closer.NextDelimiter + pc.RemoveDelimiter(closer) + closer = next + } + } + pc.ClearDelimiters(bottom) +} diff --git a/vendor/github.com/yuin/goldmark/parser/emphasis.go b/vendor/github.com/yuin/goldmark/parser/emphasis.go new file mode 100644 index 00000000..48864711 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/emphasis.go @@ -0,0 +1,50 @@ +package parser + +import ( + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" +) + +type emphasisDelimiterProcessor struct { +} + +func (p *emphasisDelimiterProcessor) IsDelimiter(b byte) bool { + return b == '*' || b == '_' +} + +func (p *emphasisDelimiterProcessor) CanOpenCloser(opener, closer *Delimiter) bool { + return opener.Char == closer.Char +} + +func (p *emphasisDelimiterProcessor) OnMatch(consumes int) ast.Node { + return ast.NewEmphasis(consumes) +} + +var defaultEmphasisDelimiterProcessor = &emphasisDelimiterProcessor{} + +type emphasisParser struct { +} + +var defaultEmphasisParser = &emphasisParser{} + +// NewEmphasisParser return a new InlineParser that parses emphasises. +func NewEmphasisParser() InlineParser { + return defaultEmphasisParser +} + +func (s *emphasisParser) Trigger() []byte { + return []byte{'*', '_'} +} + +func (s *emphasisParser) Parse(parent ast.Node, block text.Reader, pc Context) ast.Node { + before := block.PrecendingCharacter() + line, segment := block.PeekLine() + node := ScanDelimiter(line, before, 1, defaultEmphasisDelimiterProcessor) + if node == nil { + return nil + } + node.Segment = segment.WithStop(segment.Start + node.OriginalLength) + block.Advance(node.OriginalLength) + pc.PushDelimiter(node) + return node +} diff --git a/vendor/github.com/yuin/goldmark/parser/fcode_block.go b/vendor/github.com/yuin/goldmark/parser/fcode_block.go new file mode 100644 index 00000000..5914138d --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/fcode_block.go @@ -0,0 +1,121 @@ +package parser + +import ( + "bytes" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +type fencedCodeBlockParser struct { +} + +var defaultFencedCodeBlockParser = &fencedCodeBlockParser{} + +// NewFencedCodeBlockParser returns a new BlockParser that +// parses fenced code blocks. +func NewFencedCodeBlockParser() BlockParser { + return defaultFencedCodeBlockParser +} + +type fenceData struct { + char byte + indent int + length int + node ast.Node +} + +var fencedCodeBlockInfoKey = NewContextKey() + +func (b *fencedCodeBlockParser) Trigger() []byte { + return []byte{'~', '`'} +} + +func (b *fencedCodeBlockParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) { + line, segment := reader.PeekLine() + pos := pc.BlockOffset() + if pos < 0 || (line[pos] != '`' && line[pos] != '~') { + return nil, NoChildren + } + findent := pos + fenceChar := line[pos] + i := pos + for ; i < len(line) && line[i] == fenceChar; i++ { + } + oFenceLength := i - pos + if oFenceLength < 3 { + return nil, NoChildren + } + var info *ast.Text + if i < len(line)-1 { + rest := line[i:] + left := util.TrimLeftSpaceLength(rest) + right := util.TrimRightSpaceLength(rest) + if left < len(rest)-right { + infoStart, infoStop := segment.Start-segment.Padding+i+left, segment.Stop-right + value := rest[left : len(rest)-right] + if fenceChar == '`' && bytes.IndexByte(value, '`') > -1 { + return nil, NoChildren + } else if infoStart != infoStop { + info = ast.NewTextSegment(text.NewSegment(infoStart, infoStop)) + } + } + } + node := ast.NewFencedCodeBlock(info) + pc.Set(fencedCodeBlockInfoKey, &fenceData{fenceChar, findent, oFenceLength, node}) + return node, NoChildren + +} + +func (b *fencedCodeBlockParser) Continue(node ast.Node, reader text.Reader, pc Context) State { + line, segment := reader.PeekLine() + fdata := pc.Get(fencedCodeBlockInfoKey).(*fenceData) + + w, pos := util.IndentWidth(line, reader.LineOffset()) + if w < 4 { + i := pos + for ; i < len(line) && line[i] == fdata.char; i++ { + } + length := i - pos + if length >= fdata.length && util.IsBlank(line[i:]) { + newline := 1 + if line[len(line)-1] != '\n' { + newline = 0 + } + reader.Advance(segment.Stop - segment.Start - newline - segment.Padding) + return Close + } + } + pos, padding := util.IndentPositionPadding(line, reader.LineOffset(), segment.Padding, fdata.indent) + if pos < 0 { + pos = util.FirstNonSpacePosition(line) + if pos < 0 { + pos = 0 + } + padding = 0 + } + seg := text.NewSegmentPadding(segment.Start+pos, segment.Stop, padding) + // if code block line starts with a tab, keep a tab as it is. + if padding != 0 { + preserveLeadingTabInCodeBlock(&seg, reader, fdata.indent) + } + node.Lines().Append(seg) + reader.AdvanceAndSetPadding(segment.Stop-segment.Start-pos-1, padding) + return Continue | NoChildren +} + +func (b *fencedCodeBlockParser) Close(node ast.Node, reader text.Reader, pc Context) { + fdata := pc.Get(fencedCodeBlockInfoKey).(*fenceData) + if fdata.node == node { + pc.Set(fencedCodeBlockInfoKey, nil) + } +} + +func (b *fencedCodeBlockParser) CanInterruptParagraph() bool { + return true +} + +func (b *fencedCodeBlockParser) CanAcceptIndentedLine() bool { + return false +} diff --git a/vendor/github.com/yuin/goldmark/parser/html_block.go b/vendor/github.com/yuin/goldmark/parser/html_block.go new file mode 100644 index 00000000..e6f5a3fd --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/html_block.go @@ -0,0 +1,228 @@ +package parser + +import ( + "bytes" + "regexp" + "strings" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +var allowedBlockTags = map[string]bool{ + "address": true, + "article": true, + "aside": true, + "base": true, + "basefont": true, + "blockquote": true, + "body": true, + "caption": true, + "center": true, + "col": true, + "colgroup": true, + "dd": true, + "details": true, + "dialog": true, + "dir": true, + "div": true, + "dl": true, + "dt": true, + "fieldset": true, + "figcaption": true, + "figure": true, + "footer": true, + "form": true, + "frame": true, + "frameset": true, + "h1": true, + "h2": true, + "h3": true, + "h4": true, + "h5": true, + "h6": true, + "head": true, + "header": true, + "hr": true, + "html": true, + "iframe": true, + "legend": true, + "li": true, + "link": true, + "main": true, + "menu": true, + "menuitem": true, + "meta": true, + "nav": true, + "noframes": true, + "ol": true, + "optgroup": true, + "option": true, + "p": true, + "param": true, + "section": true, + "source": true, + "summary": true, + "table": true, + "tbody": true, + "td": true, + "tfoot": true, + "th": true, + "thead": true, + "title": true, + "tr": true, + "track": true, + "ul": true, +} + +var htmlBlockType1OpenRegexp = regexp.MustCompile(`(?i)^[ ]{0,3}<(script|pre|style|textarea)(?:\s.*|>.*|/>.*|)(?:\r\n|\n)?$`) +var htmlBlockType1CloseRegexp = regexp.MustCompile(`(?i)^.*</(?:script|pre|style|textarea)>.*`) + +var htmlBlockType2OpenRegexp = regexp.MustCompile(`^[ ]{0,3}<!\-\-`) +var htmlBlockType2Close = []byte{'-', '-', '>'} + +var htmlBlockType3OpenRegexp = regexp.MustCompile(`^[ ]{0,3}<\?`) +var htmlBlockType3Close = []byte{'?', '>'} + +var htmlBlockType4OpenRegexp = regexp.MustCompile(`^[ ]{0,3}<![A-Z]+.*(?:\r\n|\n)?$`) +var htmlBlockType4Close = []byte{'>'} + +var htmlBlockType5OpenRegexp = regexp.MustCompile(`^[ ]{0,3}<\!\[CDATA\[`) +var htmlBlockType5Close = []byte{']', ']', '>'} + +var htmlBlockType6Regexp = regexp.MustCompile(`^[ ]{0,3}<(?:/[ ]*)?([a-zA-Z]+[a-zA-Z0-9\-]*)(?:[ ].*|>.*|/>.*|)(?:\r\n|\n)?$`) + +var htmlBlockType7Regexp = regexp.MustCompile(`^[ ]{0,3}<(/[ ]*)?([a-zA-Z]+[a-zA-Z0-9\-]*)(` + attributePattern + `*)[ ]*(?:>|/>)[ ]*(?:\r\n|\n)?$`) + +type htmlBlockParser struct { +} + +var defaultHTMLBlockParser = &htmlBlockParser{} + +// NewHTMLBlockParser return a new BlockParser that can parse html +// blocks. +func NewHTMLBlockParser() BlockParser { + return defaultHTMLBlockParser +} + +func (b *htmlBlockParser) Trigger() []byte { + return []byte{'<'} +} + +func (b *htmlBlockParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) { + var node *ast.HTMLBlock + line, segment := reader.PeekLine() + last := pc.LastOpenedBlock().Node + if pos := pc.BlockOffset(); pos < 0 || line[pos] != '<' { + return nil, NoChildren + } + + if m := htmlBlockType1OpenRegexp.FindSubmatchIndex(line); m != nil { + node = ast.NewHTMLBlock(ast.HTMLBlockType1) + } else if htmlBlockType2OpenRegexp.Match(line) { + node = ast.NewHTMLBlock(ast.HTMLBlockType2) + } else if htmlBlockType3OpenRegexp.Match(line) { + node = ast.NewHTMLBlock(ast.HTMLBlockType3) + } else if htmlBlockType4OpenRegexp.Match(line) { + node = ast.NewHTMLBlock(ast.HTMLBlockType4) + } else if htmlBlockType5OpenRegexp.Match(line) { + node = ast.NewHTMLBlock(ast.HTMLBlockType5) + } else if match := htmlBlockType7Regexp.FindSubmatchIndex(line); match != nil { + isCloseTag := match[2] > -1 && bytes.Equal(line[match[2]:match[3]], []byte("/")) + hasAttr := match[6] != match[7] + tagName := strings.ToLower(string(line[match[4]:match[5]])) + _, ok := allowedBlockTags[tagName] + if ok { + node = ast.NewHTMLBlock(ast.HTMLBlockType6) + } else if tagName != "script" && tagName != "style" && tagName != "pre" && !ast.IsParagraph(last) && !(isCloseTag && hasAttr) { // type 7 can not interrupt paragraph + node = ast.NewHTMLBlock(ast.HTMLBlockType7) + } + } + if node == nil { + if match := htmlBlockType6Regexp.FindSubmatchIndex(line); match != nil { + tagName := string(line[match[2]:match[3]]) + _, ok := allowedBlockTags[strings.ToLower(tagName)] + if ok { + node = ast.NewHTMLBlock(ast.HTMLBlockType6) + } + } + } + if node != nil { + reader.Advance(segment.Len() - 1) + node.Lines().Append(segment) + return node, NoChildren + } + return nil, NoChildren +} + +func (b *htmlBlockParser) Continue(node ast.Node, reader text.Reader, pc Context) State { + htmlBlock := node.(*ast.HTMLBlock) + lines := htmlBlock.Lines() + line, segment := reader.PeekLine() + var closurePattern []byte + + switch htmlBlock.HTMLBlockType { + case ast.HTMLBlockType1: + if lines.Len() == 1 { + firstLine := lines.At(0) + if htmlBlockType1CloseRegexp.Match(firstLine.Value(reader.Source())) { + return Close + } + } + if htmlBlockType1CloseRegexp.Match(line) { + htmlBlock.ClosureLine = segment + reader.Advance(segment.Len() - 1) + return Close + } + case ast.HTMLBlockType2: + closurePattern = htmlBlockType2Close + fallthrough + case ast.HTMLBlockType3: + if closurePattern == nil { + closurePattern = htmlBlockType3Close + } + fallthrough + case ast.HTMLBlockType4: + if closurePattern == nil { + closurePattern = htmlBlockType4Close + } + fallthrough + case ast.HTMLBlockType5: + if closurePattern == nil { + closurePattern = htmlBlockType5Close + } + + if lines.Len() == 1 { + firstLine := lines.At(0) + if bytes.Contains(firstLine.Value(reader.Source()), closurePattern) { + return Close + } + } + if bytes.Contains(line, closurePattern) { + htmlBlock.ClosureLine = segment + reader.Advance(segment.Len() - 1) + return Close + } + + case ast.HTMLBlockType6, ast.HTMLBlockType7: + if util.IsBlank(line) { + return Close + } + } + node.Lines().Append(segment) + reader.Advance(segment.Len() - 1) + return Continue | NoChildren +} + +func (b *htmlBlockParser) Close(node ast.Node, reader text.Reader, pc Context) { + // nothing to do +} + +func (b *htmlBlockParser) CanInterruptParagraph() bool { + return true +} + +func (b *htmlBlockParser) CanAcceptIndentedLine() bool { + return false +} diff --git a/vendor/github.com/yuin/goldmark/parser/link.go b/vendor/github.com/yuin/goldmark/parser/link.go new file mode 100644 index 00000000..bd96dfb7 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/link.go @@ -0,0 +1,382 @@ +package parser + +import ( + "fmt" + "strings" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +var linkLabelStateKey = NewContextKey() + +type linkLabelState struct { + ast.BaseInline + + Segment text.Segment + + IsImage bool + + Prev *linkLabelState + + Next *linkLabelState + + First *linkLabelState + + Last *linkLabelState +} + +func newLinkLabelState(segment text.Segment, isImage bool) *linkLabelState { + return &linkLabelState{ + Segment: segment, + IsImage: isImage, + } +} + +func (s *linkLabelState) Text(source []byte) []byte { + return s.Segment.Value(source) +} + +func (s *linkLabelState) Dump(source []byte, level int) { + fmt.Printf("%slinkLabelState: \"%s\"\n", strings.Repeat(" ", level), s.Text(source)) +} + +var kindLinkLabelState = ast.NewNodeKind("LinkLabelState") + +func (s *linkLabelState) Kind() ast.NodeKind { + return kindLinkLabelState +} + +func pushLinkLabelState(pc Context, v *linkLabelState) { + tlist := pc.Get(linkLabelStateKey) + var list *linkLabelState + if tlist == nil { + list = v + v.First = v + v.Last = v + pc.Set(linkLabelStateKey, list) + } else { + list = tlist.(*linkLabelState) + l := list.Last + list.Last = v + l.Next = v + v.Prev = l + } +} + +func removeLinkLabelState(pc Context, d *linkLabelState) { + tlist := pc.Get(linkLabelStateKey) + var list *linkLabelState + if tlist == nil { + return + } + list = tlist.(*linkLabelState) + + if d.Prev == nil { + list = d.Next + if list != nil { + list.First = d + list.Last = d.Last + list.Prev = nil + pc.Set(linkLabelStateKey, list) + } else { + pc.Set(linkLabelStateKey, nil) + } + } else { + d.Prev.Next = d.Next + if d.Next != nil { + d.Next.Prev = d.Prev + } + } + if list != nil && d.Next == nil { + list.Last = d.Prev + } + d.Next = nil + d.Prev = nil + d.First = nil + d.Last = nil +} + +type linkParser struct { +} + +var defaultLinkParser = &linkParser{} + +// NewLinkParser return a new InlineParser that parses links. +func NewLinkParser() InlineParser { + return defaultLinkParser +} + +func (s *linkParser) Trigger() []byte { + return []byte{'!', '[', ']'} +} + +var linkBottom = NewContextKey() + +func (s *linkParser) Parse(parent ast.Node, block text.Reader, pc Context) ast.Node { + line, segment := block.PeekLine() + if line[0] == '!' { + if len(line) > 1 && line[1] == '[' { + block.Advance(1) + pc.Set(linkBottom, pc.LastDelimiter()) + return processLinkLabelOpen(block, segment.Start+1, true, pc) + } + return nil + } + if line[0] == '[' { + pc.Set(linkBottom, pc.LastDelimiter()) + return processLinkLabelOpen(block, segment.Start, false, pc) + } + + // line[0] == ']' + tlist := pc.Get(linkLabelStateKey) + if tlist == nil { + return nil + } + last := tlist.(*linkLabelState).Last + if last == nil { + return nil + } + block.Advance(1) + removeLinkLabelState(pc, last) + if !last.IsImage && s.containsLink(last) { // a link in a link text is not allowed + ast.MergeOrReplaceTextSegment(last.Parent(), last, last.Segment) + return nil + } + + c := block.Peek() + l, pos := block.Position() + var link *ast.Link + var hasValue bool + if c == '(' { // normal link + link = s.parseLink(parent, last, block, pc) + } else if c == '[' { // reference link + link, hasValue = s.parseReferenceLink(parent, last, block, pc) + if link == nil && hasValue { + ast.MergeOrReplaceTextSegment(last.Parent(), last, last.Segment) + return nil + } + } + + if link == nil { + // maybe shortcut reference link + block.SetPosition(l, pos) + ssegment := text.NewSegment(last.Segment.Stop, segment.Start) + maybeReference := block.Value(ssegment) + ref, ok := pc.Reference(util.ToLinkReference(maybeReference)) + if !ok { + ast.MergeOrReplaceTextSegment(last.Parent(), last, last.Segment) + return nil + } + link = ast.NewLink() + s.processLinkLabel(parent, link, last, pc) + link.Title = ref.Title() + link.Destination = ref.Destination() + } + if last.IsImage { + last.Parent().RemoveChild(last.Parent(), last) + return ast.NewImage(link) + } + last.Parent().RemoveChild(last.Parent(), last) + return link +} + +func (s *linkParser) containsLink(n ast.Node) bool { + if n == nil { + return false + } + for c := n; c != nil; c = c.NextSibling() { + if _, ok := c.(*ast.Link); ok { + return true + } + if s.containsLink(c.FirstChild()) { + return true + } + } + return false +} + +func processLinkLabelOpen(block text.Reader, pos int, isImage bool, pc Context) *linkLabelState { + start := pos + if isImage { + start-- + } + state := newLinkLabelState(text.NewSegment(start, pos+1), isImage) + pushLinkLabelState(pc, state) + block.Advance(1) + return state +} + +func (s *linkParser) processLinkLabel(parent ast.Node, link *ast.Link, last *linkLabelState, pc Context) { + var bottom ast.Node + if v := pc.Get(linkBottom); v != nil { + bottom = v.(ast.Node) + } + pc.Set(linkBottom, nil) + ProcessDelimiters(bottom, pc) + for c := last.NextSibling(); c != nil; { + next := c.NextSibling() + parent.RemoveChild(parent, c) + link.AppendChild(link, c) + c = next + } +} + +var linkFindClosureOptions text.FindClosureOptions = text.FindClosureOptions{ + Nesting: false, + Newline: true, + Advance: true, +} + +func (s *linkParser) parseReferenceLink(parent ast.Node, last *linkLabelState, block text.Reader, pc Context) (*ast.Link, bool) { + _, orgpos := block.Position() + block.Advance(1) // skip '[' + segments, found := block.FindClosure('[', ']', linkFindClosureOptions) + if !found { + return nil, false + } + + var maybeReference []byte + if segments.Len() == 1 { // avoid allocate a new byte slice + maybeReference = block.Value(segments.At(0)) + } else { + maybeReference = []byte{} + for i := 0; i < segments.Len(); i++ { + s := segments.At(i) + maybeReference = append(maybeReference, block.Value(s)...) + } + } + if util.IsBlank(maybeReference) { // collapsed reference link + s := text.NewSegment(last.Segment.Stop, orgpos.Start-1) + maybeReference = block.Value(s) + } + + ref, ok := pc.Reference(util.ToLinkReference(maybeReference)) + if !ok { + return nil, true + } + + link := ast.NewLink() + s.processLinkLabel(parent, link, last, pc) + link.Title = ref.Title() + link.Destination = ref.Destination() + return link, true +} + +func (s *linkParser) parseLink(parent ast.Node, last *linkLabelState, block text.Reader, pc Context) *ast.Link { + block.Advance(1) // skip '(' + block.SkipSpaces() + var title []byte + var destination []byte + var ok bool + if block.Peek() == ')' { // empty link like '[link]()' + block.Advance(1) + } else { + destination, ok = parseLinkDestination(block) + if !ok { + return nil + } + block.SkipSpaces() + if block.Peek() == ')' { + block.Advance(1) + } else { + title, ok = parseLinkTitle(block) + if !ok { + return nil + } + block.SkipSpaces() + if block.Peek() == ')' { + block.Advance(1) + } else { + return nil + } + } + } + + link := ast.NewLink() + s.processLinkLabel(parent, link, last, pc) + link.Destination = destination + link.Title = title + return link +} + +func parseLinkDestination(block text.Reader) ([]byte, bool) { + block.SkipSpaces() + line, _ := block.PeekLine() + if block.Peek() == '<' { + i := 1 + for i < len(line) { + c := line[i] + if c == '\\' && i < len(line)-1 && util.IsPunct(line[i+1]) { + i += 2 + continue + } else if c == '>' { + block.Advance(i + 1) + return line[1:i], true + } + i++ + } + return nil, false + } + opened := 0 + i := 0 + for i < len(line) { + c := line[i] + if c == '\\' && i < len(line)-1 && util.IsPunct(line[i+1]) { + i += 2 + continue + } else if c == '(' { + opened++ + } else if c == ')' { + opened-- + if opened < 0 { + break + } + } else if util.IsSpace(c) { + break + } + i++ + } + block.Advance(i) + return line[:i], len(line[:i]) != 0 +} + +func parseLinkTitle(block text.Reader) ([]byte, bool) { + block.SkipSpaces() + opener := block.Peek() + if opener != '"' && opener != '\'' && opener != '(' { + return nil, false + } + closer := opener + if opener == '(' { + closer = ')' + } + block.Advance(1) + segments, found := block.FindClosure(opener, closer, linkFindClosureOptions) + if found { + if segments.Len() == 1 { + return block.Value(segments.At(0)), true + } + var title []byte + for i := 0; i < segments.Len(); i++ { + s := segments.At(i) + title = append(title, block.Value(s)...) + } + return title, true + } + return nil, false +} + +func (s *linkParser) CloseBlock(parent ast.Node, block text.Reader, pc Context) { + tlist := pc.Get(linkLabelStateKey) + if tlist == nil { + return + } + for s := tlist.(*linkLabelState); s != nil; { + next := s.Next + removeLinkLabelState(pc, s) + s.Parent().ReplaceChild(s.Parent(), s, ast.NewTextSegment(s.Segment)) + s = next + } +} diff --git a/vendor/github.com/yuin/goldmark/parser/link_ref.go b/vendor/github.com/yuin/goldmark/parser/link_ref.go new file mode 100644 index 00000000..ea3f6544 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/link_ref.go @@ -0,0 +1,152 @@ +package parser + +import ( + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +type linkReferenceParagraphTransformer struct { +} + +// LinkReferenceParagraphTransformer is a ParagraphTransformer implementation +// that parses and extracts link reference from paragraphs. +var LinkReferenceParagraphTransformer = &linkReferenceParagraphTransformer{} + +func (p *linkReferenceParagraphTransformer) Transform(node *ast.Paragraph, reader text.Reader, pc Context) { + lines := node.Lines() + block := text.NewBlockReader(reader.Source(), lines) + removes := [][2]int{} + for { + start, end := parseLinkReferenceDefinition(block, pc) + if start > -1 { + if start == end { + end++ + } + removes = append(removes, [2]int{start, end}) + continue + } + break + } + + offset := 0 + for _, remove := range removes { + if lines.Len() == 0 { + break + } + s := lines.Sliced(remove[1]-offset, lines.Len()) + lines.SetSliced(0, remove[0]-offset) + lines.AppendAll(s) + offset = remove[1] + } + + if lines.Len() == 0 { + t := ast.NewTextBlock() + t.SetBlankPreviousLines(node.HasBlankPreviousLines()) + node.Parent().ReplaceChild(node.Parent(), node, t) + return + } + + node.SetLines(lines) +} + +func parseLinkReferenceDefinition(block text.Reader, pc Context) (int, int) { + block.SkipSpaces() + line, _ := block.PeekLine() + if line == nil { + return -1, -1 + } + startLine, _ := block.Position() + width, pos := util.IndentWidth(line, 0) + if width > 3 { + return -1, -1 + } + if width != 0 { + pos++ + } + if line[pos] != '[' { + return -1, -1 + } + block.Advance(pos + 1) + segments, found := block.FindClosure('[', ']', linkFindClosureOptions) + if !found { + return -1, -1 + } + var label []byte + if segments.Len() == 1 { + label = block.Value(segments.At(0)) + } else { + for i := 0; i < segments.Len(); i++ { + s := segments.At(i) + label = append(label, block.Value(s)...) + } + } + if util.IsBlank(label) { + return -1, -1 + } + if block.Peek() != ':' { + return -1, -1 + } + block.Advance(1) + block.SkipSpaces() + destination, ok := parseLinkDestination(block) + if !ok { + return -1, -1 + } + line, _ = block.PeekLine() + isNewLine := line == nil || util.IsBlank(line) + + endLine, _ := block.Position() + _, spaces, _ := block.SkipSpaces() + opener := block.Peek() + if opener != '"' && opener != '\'' && opener != '(' { + if !isNewLine { + return -1, -1 + } + ref := NewReference(label, destination, nil) + pc.AddReference(ref) + return startLine, endLine + 1 + } + if spaces == 0 { + return -1, -1 + } + block.Advance(1) + closer := opener + if opener == '(' { + closer = ')' + } + segments, found = block.FindClosure(opener, closer, linkFindClosureOptions) + if !found { + if !isNewLine { + return -1, -1 + } + ref := NewReference(label, destination, nil) + pc.AddReference(ref) + block.AdvanceLine() + return startLine, endLine + 1 + } + var title []byte + if segments.Len() == 1 { + title = block.Value(segments.At(0)) + } else { + for i := 0; i < segments.Len(); i++ { + s := segments.At(i) + title = append(title, block.Value(s)...) + } + } + + line, _ = block.PeekLine() + if line != nil && !util.IsBlank(line) { + if !isNewLine { + return -1, -1 + } + ref := NewReference(label, destination, title) + pc.AddReference(ref) + return startLine, endLine + } + + endLine, _ = block.Position() + ref := NewReference(label, destination, title) + pc.AddReference(ref) + return startLine, endLine + 1 +} diff --git a/vendor/github.com/yuin/goldmark/parser/list.go b/vendor/github.com/yuin/goldmark/parser/list.go new file mode 100644 index 00000000..2a1c03a9 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/list.go @@ -0,0 +1,283 @@ +package parser + +import ( + "strconv" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +type listItemType int + +const ( + notList listItemType = iota + bulletList + orderedList +) + +var skipListParserKey = NewContextKey() +var emptyListItemWithBlankLines = NewContextKey() +var listItemFlagValue interface{} = true + +// Same as +// `^(([ ]*)([\-\*\+]))(\s+.*)?\n?$`.FindSubmatchIndex or +// `^(([ ]*)(\d{1,9}[\.\)]))(\s+.*)?\n?$`.FindSubmatchIndex +func parseListItem(line []byte) ([6]int, listItemType) { + i := 0 + l := len(line) + ret := [6]int{} + for ; i < l && line[i] == ' '; i++ { + c := line[i] + if c == '\t' { + return ret, notList + } + } + if i > 3 { + return ret, notList + } + ret[0] = 0 + ret[1] = i + ret[2] = i + var typ listItemType + if i < l && (line[i] == '-' || line[i] == '*' || line[i] == '+') { + i++ + ret[3] = i + typ = bulletList + } else if i < l { + for ; i < l && util.IsNumeric(line[i]); i++ { + } + ret[3] = i + if ret[3] == ret[2] || ret[3]-ret[2] > 9 { + return ret, notList + } + if i < l && (line[i] == '.' || line[i] == ')') { + i++ + ret[3] = i + } else { + return ret, notList + } + typ = orderedList + } else { + return ret, notList + } + if i < l && line[i] != '\n' { + w, _ := util.IndentWidth(line[i:], 0) + if w == 0 { + return ret, notList + } + } + if i >= l { + ret[4] = -1 + ret[5] = -1 + return ret, typ + } + ret[4] = i + ret[5] = len(line) + if line[ret[5]-1] == '\n' && line[i] != '\n' { + ret[5]-- + } + return ret, typ +} + +func matchesListItem(source []byte, strict bool) ([6]int, listItemType) { + m, typ := parseListItem(source) + if typ != notList && (!strict || strict && m[1] < 4) { + return m, typ + } + return m, notList +} + +func calcListOffset(source []byte, match [6]int) int { + offset := 0 + if match[4] < 0 || util.IsBlank(source[match[4]:]) { // list item starts with a blank line + offset = 1 + } else { + offset, _ = util.IndentWidth(source[match[4]:], match[4]) + if offset > 4 { // offseted codeblock + offset = 1 + } + } + return offset +} + +func lastOffset(node ast.Node) int { + lastChild := node.LastChild() + if lastChild != nil { + return lastChild.(*ast.ListItem).Offset + } + return 0 +} + +type listParser struct { +} + +var defaultListParser = &listParser{} + +// NewListParser returns a new BlockParser that +// parses lists. +// This parser must take precedence over the ListItemParser. +func NewListParser() BlockParser { + return defaultListParser +} + +func (b *listParser) Trigger() []byte { + return []byte{'-', '+', '*', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'} +} + +func (b *listParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) { + last := pc.LastOpenedBlock().Node + if _, lok := last.(*ast.List); lok || pc.Get(skipListParserKey) != nil { + pc.Set(skipListParserKey, nil) + return nil, NoChildren + } + line, _ := reader.PeekLine() + match, typ := matchesListItem(line, true) + if typ == notList { + return nil, NoChildren + } + start := -1 + if typ == orderedList { + number := line[match[2] : match[3]-1] + start, _ = strconv.Atoi(string(number)) + } + + if ast.IsParagraph(last) && last.Parent() == parent { + // we allow only lists starting with 1 to interrupt paragraphs. + if typ == orderedList && start != 1 { + return nil, NoChildren + } + //an empty list item cannot interrupt a paragraph: + if match[4] < 0 || util.IsBlank(line[match[4]:match[5]]) { + return nil, NoChildren + } + } + + marker := line[match[3]-1] + node := ast.NewList(marker) + if start > -1 { + node.Start = start + } + pc.Set(emptyListItemWithBlankLines, nil) + return node, HasChildren +} + +func (b *listParser) Continue(node ast.Node, reader text.Reader, pc Context) State { + list := node.(*ast.List) + line, _ := reader.PeekLine() + if util.IsBlank(line) { + if node.LastChild().ChildCount() == 0 { + pc.Set(emptyListItemWithBlankLines, listItemFlagValue) + } + return Continue | HasChildren + } + + // "offset" means a width that bar indicates. + // - aaaaaaaa + // |----| + // + // If the indent is less than the last offset like + // - a + // - b <--- current line + // it maybe a new child of the list. + // + // Empty list items can have multiple blanklines + // + // - <--- 1st item is an empty thus "offset" is unknown + // + // + // - <--- current line + // + // -> 1 list with 2 blank items + // + // So if the last item is an empty, it maybe a new child of the list. + // + offset := lastOffset(node) + lastIsEmpty := node.LastChild().ChildCount() == 0 + indent, _ := util.IndentWidth(line, reader.LineOffset()) + + if indent < offset || lastIsEmpty { + if indent < 4 { + match, typ := matchesListItem(line, false) // may have a leading spaces more than 3 + if typ != notList && match[1]-offset < 4 { + marker := line[match[3]-1] + if !list.CanContinue(marker, typ == orderedList) { + return Close + } + // Thematic Breaks take precedence over lists + if isThematicBreak(line[match[3]-1:], 0) { + isHeading := false + last := pc.LastOpenedBlock().Node + if ast.IsParagraph(last) { + c, ok := matchesSetextHeadingBar(line[match[3]-1:]) + if ok && c == '-' { + isHeading = true + } + } + if !isHeading { + return Close + } + } + return Continue | HasChildren + } + } + if !lastIsEmpty { + return Close + } + } + + // Non empty items can not exist next to an empty list item + // with blank lines. So we need to close the current list + // + // - + // + // foo + // + // -> 1 list with 1 blank items and 1 paragraph + if pc.Get(emptyListItemWithBlankLines) != nil { + return Close + } + return Continue | HasChildren +} + +func (b *listParser) Close(node ast.Node, reader text.Reader, pc Context) { + list := node.(*ast.List) + + for c := node.FirstChild(); c != nil && list.IsTight; c = c.NextSibling() { + if c.FirstChild() != nil && c.FirstChild() != c.LastChild() { + for c1 := c.FirstChild().NextSibling(); c1 != nil; c1 = c1.NextSibling() { + if bl, ok := c1.(ast.Node); ok && bl.HasBlankPreviousLines() { + list.IsTight = false + break + } + } + } + if c != node.FirstChild() { + if bl, ok := c.(ast.Node); ok && bl.HasBlankPreviousLines() { + list.IsTight = false + } + } + } + + if list.IsTight { + for child := node.FirstChild(); child != nil; child = child.NextSibling() { + for gc := child.FirstChild(); gc != nil; { + paragraph, ok := gc.(*ast.Paragraph) + gc = gc.NextSibling() + if ok { + textBlock := ast.NewTextBlock() + textBlock.SetLines(paragraph.Lines()) + child.ReplaceChild(child, paragraph, textBlock) + } + } + } + } +} + +func (b *listParser) CanInterruptParagraph() bool { + return true +} + +func (b *listParser) CanAcceptIndentedLine() bool { + return false +} diff --git a/vendor/github.com/yuin/goldmark/parser/list_item.go b/vendor/github.com/yuin/goldmark/parser/list_item.go new file mode 100644 index 00000000..81357a9a --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/list_item.go @@ -0,0 +1,90 @@ +package parser + +import ( + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +type listItemParser struct { +} + +var defaultListItemParser = &listItemParser{} + +// NewListItemParser returns a new BlockParser that +// parses list items. +func NewListItemParser() BlockParser { + return defaultListItemParser +} + +func (b *listItemParser) Trigger() []byte { + return []byte{'-', '+', '*', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'} +} + +func (b *listItemParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) { + list, lok := parent.(*ast.List) + if !lok { // list item must be a child of a list + return nil, NoChildren + } + offset := lastOffset(list) + line, _ := reader.PeekLine() + match, typ := matchesListItem(line, false) + if typ == notList { + return nil, NoChildren + } + if match[1]-offset > 3 { + return nil, NoChildren + } + + pc.Set(emptyListItemWithBlankLines, nil) + + itemOffset := calcListOffset(line, match) + node := ast.NewListItem(match[3] + itemOffset) + if match[4] < 0 || util.IsBlank(line[match[4]:match[5]]) { + return node, NoChildren + } + + pos, padding := util.IndentPosition(line[match[4]:], match[4], itemOffset) + child := match[3] + pos + reader.AdvanceAndSetPadding(child, padding) + return node, HasChildren +} + +func (b *listItemParser) Continue(node ast.Node, reader text.Reader, pc Context) State { + line, _ := reader.PeekLine() + if util.IsBlank(line) { + reader.Advance(len(line) - 1) + return Continue | HasChildren + } + + offset := lastOffset(node.Parent()) + isEmpty := node.ChildCount() == 0 + indent, _ := util.IndentWidth(line, reader.LineOffset()) + if (isEmpty || indent < offset) && indent < 4 { + _, typ := matchesListItem(line, true) + // new list item found + if typ != notList { + pc.Set(skipListParserKey, listItemFlagValue) + return Close + } + if !isEmpty { + return Close + } + } + pos, padding := util.IndentPosition(line, reader.LineOffset(), offset) + reader.AdvanceAndSetPadding(pos, padding) + + return Continue | HasChildren +} + +func (b *listItemParser) Close(node ast.Node, reader text.Reader, pc Context) { + // nothing to do +} + +func (b *listItemParser) CanInterruptParagraph() bool { + return true +} + +func (b *listItemParser) CanAcceptIndentedLine() bool { + return false +} diff --git a/vendor/github.com/yuin/goldmark/parser/paragraph.go b/vendor/github.com/yuin/goldmark/parser/paragraph.go new file mode 100644 index 00000000..2dd2b9a9 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/paragraph.go @@ -0,0 +1,71 @@ +package parser + +import ( + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" +) + +type paragraphParser struct { +} + +var defaultParagraphParser = ¶graphParser{} + +// NewParagraphParser returns a new BlockParser that +// parses paragraphs. +func NewParagraphParser() BlockParser { + return defaultParagraphParser +} + +func (b *paragraphParser) Trigger() []byte { + return nil +} + +func (b *paragraphParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) { + _, segment := reader.PeekLine() + segment = segment.TrimLeftSpace(reader.Source()) + if segment.IsEmpty() { + return nil, NoChildren + } + node := ast.NewParagraph() + node.Lines().Append(segment) + reader.Advance(segment.Len() - 1) + return node, NoChildren +} + +func (b *paragraphParser) Continue(node ast.Node, reader text.Reader, pc Context) State { + _, segment := reader.PeekLine() + segment = segment.TrimLeftSpace(reader.Source()) + if segment.IsEmpty() { + return Close + } + node.Lines().Append(segment) + reader.Advance(segment.Len() - 1) + return Continue | NoChildren +} + +func (b *paragraphParser) Close(node ast.Node, reader text.Reader, pc Context) { + parent := node.Parent() + if parent == nil { + // paragraph has been transformed + return + } + lines := node.Lines() + if lines.Len() != 0 { + // trim trailing spaces + length := lines.Len() + lastLine := node.Lines().At(length - 1) + node.Lines().Set(length-1, lastLine.TrimRightSpace(reader.Source())) + } + if lines.Len() == 0 { + node.Parent().RemoveChild(node.Parent(), node) + return + } +} + +func (b *paragraphParser) CanInterruptParagraph() bool { + return false +} + +func (b *paragraphParser) CanAcceptIndentedLine() bool { + return false +} diff --git a/vendor/github.com/yuin/goldmark/parser/parser.go b/vendor/github.com/yuin/goldmark/parser/parser.go new file mode 100644 index 00000000..bac07045 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/parser.go @@ -0,0 +1,1236 @@ +// Package parser contains stuff that are related to parsing a Markdown text. +package parser + +import ( + "fmt" + "strings" + "sync" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +// A Reference interface represents a link reference in Markdown text. +type Reference interface { + // String implements Stringer. + String() string + + // Label returns a label of the reference. + Label() []byte + + // Destination returns a destination(URL) of the reference. + Destination() []byte + + // Title returns a title of the reference. + Title() []byte +} + +type reference struct { + label []byte + destination []byte + title []byte +} + +// NewReference returns a new Reference. +func NewReference(label, destination, title []byte) Reference { + return &reference{label, destination, title} +} + +func (r *reference) Label() []byte { + return r.label +} + +func (r *reference) Destination() []byte { + return r.destination +} + +func (r *reference) Title() []byte { + return r.title +} + +func (r *reference) String() string { + return fmt.Sprintf("Reference{Label:%s, Destination:%s, Title:%s}", r.label, r.destination, r.title) +} + +// An IDs interface is a collection of the element ids. +type IDs interface { + // Generate generates a new element id. + Generate(value []byte, kind ast.NodeKind) []byte + + // Put puts a given element id to the used ids table. + Put(value []byte) +} + +type ids struct { + values map[string]bool +} + +func newIDs() IDs { + return &ids{ + values: map[string]bool{}, + } +} + +func (s *ids) Generate(value []byte, kind ast.NodeKind) []byte { + value = util.TrimLeftSpace(value) + value = util.TrimRightSpace(value) + result := []byte{} + for i := 0; i < len(value); { + v := value[i] + l := util.UTF8Len(v) + i += int(l) + if l != 1 { + continue + } + if util.IsAlphaNumeric(v) { + if 'A' <= v && v <= 'Z' { + v += 'a' - 'A' + } + result = append(result, v) + } else if util.IsSpace(v) || v == '-' || v == '_' { + result = append(result, '-') + } + } + if len(result) == 0 { + if kind == ast.KindHeading { + result = []byte("heading") + } else { + result = []byte("id") + } + } + if _, ok := s.values[util.BytesToReadOnlyString(result)]; !ok { + s.values[util.BytesToReadOnlyString(result)] = true + return result + } + for i := 1; ; i++ { + newResult := fmt.Sprintf("%s-%d", result, i) + if _, ok := s.values[newResult]; !ok { + s.values[newResult] = true + return []byte(newResult) + } + + } +} + +func (s *ids) Put(value []byte) { + s.values[util.BytesToReadOnlyString(value)] = true +} + +// ContextKey is a key that is used to set arbitrary values to the context. +type ContextKey int + +// ContextKeyMax is a maximum value of the ContextKey. +var ContextKeyMax ContextKey + +// NewContextKey return a new ContextKey value. +func NewContextKey() ContextKey { + ContextKeyMax++ + return ContextKeyMax +} + +// A Context interface holds a information that are necessary to parse +// Markdown text. +type Context interface { + // String implements Stringer. + String() string + + // Get returns a value associated with the given key. + Get(ContextKey) interface{} + + // ComputeIfAbsent computes a value if a value associated with the given key is absent and returns the value. + ComputeIfAbsent(ContextKey, func() interface{}) interface{} + + // Set sets the given value to the context. + Set(ContextKey, interface{}) + + // AddReference adds the given reference to this context. + AddReference(Reference) + + // Reference returns (a reference, true) if a reference associated with + // the given label exists, otherwise (nil, false). + Reference(label string) (Reference, bool) + + // References returns a list of references. + References() []Reference + + // IDs returns a collection of the element ids. + IDs() IDs + + // BlockOffset returns a first non-space character position on current line. + // This value is valid only for BlockParser.Open. + // BlockOffset returns -1 if current line is blank. + BlockOffset() int + + // BlockOffset sets a first non-space character position on current line. + // This value is valid only for BlockParser.Open. + SetBlockOffset(int) + + // BlockIndent returns an indent width on current line. + // This value is valid only for BlockParser.Open. + // BlockIndent returns -1 if current line is blank. + BlockIndent() int + + // BlockIndent sets an indent width on current line. + // This value is valid only for BlockParser.Open. + SetBlockIndent(int) + + // FirstDelimiter returns a first delimiter of the current delimiter list. + FirstDelimiter() *Delimiter + + // LastDelimiter returns a last delimiter of the current delimiter list. + LastDelimiter() *Delimiter + + // PushDelimiter appends the given delimiter to the tail of the current + // delimiter list. + PushDelimiter(delimiter *Delimiter) + + // RemoveDelimiter removes the given delimiter from the current delimiter list. + RemoveDelimiter(d *Delimiter) + + // ClearDelimiters clears the current delimiter list. + ClearDelimiters(bottom ast.Node) + + // OpenedBlocks returns a list of nodes that are currently in parsing. + OpenedBlocks() []Block + + // SetOpenedBlocks sets a list of nodes that are currently in parsing. + SetOpenedBlocks([]Block) + + // LastOpenedBlock returns a last node that is currently in parsing. + LastOpenedBlock() Block + + // IsInLinkLabel returns true if current position seems to be in link label. + IsInLinkLabel() bool +} + +// A ContextConfig struct is a data structure that holds configuration of the Context. +type ContextConfig struct { + IDs IDs +} + +// An ContextOption is a functional option type for the Context. +type ContextOption func(*ContextConfig) + +// WithIDs is a functional option for the Context. +func WithIDs(ids IDs) ContextOption { + return func(c *ContextConfig) { + c.IDs = ids + } +} + +type parseContext struct { + store []interface{} + ids IDs + refs map[string]Reference + blockOffset int + blockIndent int + delimiters *Delimiter + lastDelimiter *Delimiter + openedBlocks []Block +} + +// NewContext returns a new Context. +func NewContext(options ...ContextOption) Context { + cfg := &ContextConfig{ + IDs: newIDs(), + } + for _, option := range options { + option(cfg) + } + + return &parseContext{ + store: make([]interface{}, ContextKeyMax+1), + refs: map[string]Reference{}, + ids: cfg.IDs, + blockOffset: -1, + blockIndent: -1, + delimiters: nil, + lastDelimiter: nil, + openedBlocks: []Block{}, + } +} + +func (p *parseContext) Get(key ContextKey) interface{} { + return p.store[key] +} + +func (p *parseContext) ComputeIfAbsent(key ContextKey, f func() interface{}) interface{} { + v := p.store[key] + if v == nil { + v = f() + p.store[key] = v + } + return v +} + +func (p *parseContext) Set(key ContextKey, value interface{}) { + p.store[key] = value +} + +func (p *parseContext) IDs() IDs { + return p.ids +} + +func (p *parseContext) BlockOffset() int { + return p.blockOffset +} + +func (p *parseContext) SetBlockOffset(v int) { + p.blockOffset = v +} + +func (p *parseContext) BlockIndent() int { + return p.blockIndent +} + +func (p *parseContext) SetBlockIndent(v int) { + p.blockIndent = v +} + +func (p *parseContext) LastDelimiter() *Delimiter { + return p.lastDelimiter +} + +func (p *parseContext) FirstDelimiter() *Delimiter { + return p.delimiters +} + +func (p *parseContext) PushDelimiter(d *Delimiter) { + if p.delimiters == nil { + p.delimiters = d + p.lastDelimiter = d + } else { + l := p.lastDelimiter + p.lastDelimiter = d + l.NextDelimiter = d + d.PreviousDelimiter = l + } +} + +func (p *parseContext) RemoveDelimiter(d *Delimiter) { + if d.PreviousDelimiter == nil { + p.delimiters = d.NextDelimiter + } else { + d.PreviousDelimiter.NextDelimiter = d.NextDelimiter + if d.NextDelimiter != nil { + d.NextDelimiter.PreviousDelimiter = d.PreviousDelimiter + } + } + if d.NextDelimiter == nil { + p.lastDelimiter = d.PreviousDelimiter + } + if p.delimiters != nil { + p.delimiters.PreviousDelimiter = nil + } + if p.lastDelimiter != nil { + p.lastDelimiter.NextDelimiter = nil + } + d.NextDelimiter = nil + d.PreviousDelimiter = nil + if d.Length != 0 { + ast.MergeOrReplaceTextSegment(d.Parent(), d, d.Segment) + } else { + d.Parent().RemoveChild(d.Parent(), d) + } +} + +func (p *parseContext) ClearDelimiters(bottom ast.Node) { + if p.lastDelimiter == nil { + return + } + var c ast.Node + for c = p.lastDelimiter; c != nil && c != bottom; { + prev := c.PreviousSibling() + if d, ok := c.(*Delimiter); ok { + p.RemoveDelimiter(d) + } + c = prev + } +} + +func (p *parseContext) AddReference(ref Reference) { + key := util.ToLinkReference(ref.Label()) + if _, ok := p.refs[key]; !ok { + p.refs[key] = ref + } +} + +func (p *parseContext) Reference(label string) (Reference, bool) { + v, ok := p.refs[label] + return v, ok +} + +func (p *parseContext) References() []Reference { + ret := make([]Reference, 0, len(p.refs)) + for _, v := range p.refs { + ret = append(ret, v) + } + return ret +} + +func (p *parseContext) String() string { + refs := []string{} + for _, r := range p.refs { + refs = append(refs, r.String()) + } + + return fmt.Sprintf("Context{Store:%#v, Refs:%s}", p.store, strings.Join(refs, ",")) +} + +func (p *parseContext) OpenedBlocks() []Block { + return p.openedBlocks +} + +func (p *parseContext) SetOpenedBlocks(v []Block) { + p.openedBlocks = v +} + +func (p *parseContext) LastOpenedBlock() Block { + if l := len(p.openedBlocks); l != 0 { + return p.openedBlocks[l-1] + } + return Block{} +} + +func (p *parseContext) IsInLinkLabel() bool { + tlist := p.Get(linkLabelStateKey) + return tlist != nil +} + +// State represents parser's state. +// State is designed to use as a bit flag. +type State int + +const ( + none State = 1 << iota + + // Continue indicates parser can continue parsing. + Continue + + // Close indicates parser cannot parse anymore. + Close + + // HasChildren indicates parser may have child blocks. + HasChildren + + // NoChildren indicates parser does not have child blocks. + NoChildren + + // RequireParagraph indicates parser requires that the last node + // must be a paragraph and is not converted to other nodes by + // ParagraphTransformers. + RequireParagraph +) + +// A Config struct is a data structure that holds configuration of the Parser. +type Config struct { + Options map[OptionName]interface{} + BlockParsers util.PrioritizedSlice /*<BlockParser>*/ + InlineParsers util.PrioritizedSlice /*<InlineParser>*/ + ParagraphTransformers util.PrioritizedSlice /*<ParagraphTransformer>*/ + ASTTransformers util.PrioritizedSlice /*<ASTTransformer>*/ +} + +// NewConfig returns a new Config. +func NewConfig() *Config { + return &Config{ + Options: map[OptionName]interface{}{}, + BlockParsers: util.PrioritizedSlice{}, + InlineParsers: util.PrioritizedSlice{}, + ParagraphTransformers: util.PrioritizedSlice{}, + ASTTransformers: util.PrioritizedSlice{}, + } +} + +// An Option interface is a functional option type for the Parser. +type Option interface { + SetParserOption(*Config) +} + +// OptionName is a name of parser options. +type OptionName string + +// Attribute is an option name that spacify attributes of elements. +const optAttribute OptionName = "Attribute" + +type withAttribute struct { +} + +func (o *withAttribute) SetParserOption(c *Config) { + c.Options[optAttribute] = true +} + +// WithAttribute is a functional option that enables custom attributes. +func WithAttribute() Option { + return &withAttribute{} +} + +// A Parser interface parses Markdown text into AST nodes. +type Parser interface { + // Parse parses the given Markdown text into AST nodes. + Parse(reader text.Reader, opts ...ParseOption) ast.Node + + // AddOption adds the given option to this parser. + AddOptions(...Option) +} + +// A SetOptioner interface sets the given option to the object. +type SetOptioner interface { + // SetOption sets the given option to the object. + // Unacceptable options may be passed. + // Thus implementations must ignore unacceptable options. + SetOption(name OptionName, value interface{}) +} + +// A BlockParser interface parses a block level element like Paragraph, List, +// Blockquote etc. +type BlockParser interface { + // Trigger returns a list of characters that triggers Parse method of + // this parser. + // If Trigger returns a nil, Open will be called with any lines. + Trigger() []byte + + // Open parses the current line and returns a result of parsing. + // + // Open must not parse beyond the current line. + // If Open has been able to parse the current line, Open must advance a reader + // position by consumed byte length. + // + // If Open has not been able to parse the current line, Open should returns + // (nil, NoChildren). If Open has been able to parse the current line, Open + // should returns a new Block node and returns HasChildren or NoChildren. + Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) + + // Continue parses the current line and returns a result of parsing. + // + // Continue must not parse beyond the current line. + // If Continue has been able to parse the current line, Continue must advance + // a reader position by consumed byte length. + // + // If Continue has not been able to parse the current line, Continue should + // returns Close. If Continue has been able to parse the current line, + // Continue should returns (Continue | NoChildren) or + // (Continue | HasChildren) + Continue(node ast.Node, reader text.Reader, pc Context) State + + // Close will be called when the parser returns Close. + Close(node ast.Node, reader text.Reader, pc Context) + + // CanInterruptParagraph returns true if the parser can interrupt paragraphs, + // otherwise false. + CanInterruptParagraph() bool + + // CanAcceptIndentedLine returns true if the parser can open new node when + // the given line is being indented more than 3 spaces. + CanAcceptIndentedLine() bool +} + +// An InlineParser interface parses an inline level element like CodeSpan, Link etc. +type InlineParser interface { + // Trigger returns a list of characters that triggers Parse method of + // this parser. + // Trigger characters must be a punctuation or a halfspace. + // Halfspaces triggers this parser when character is any spaces characters or + // a head of line + Trigger() []byte + + // Parse parse the given block into an inline node. + // + // Parse can parse beyond the current line. + // If Parse has been able to parse the current line, it must advance a reader + // position by consumed byte length. + Parse(parent ast.Node, block text.Reader, pc Context) ast.Node +} + +// A CloseBlocker interface is a callback function that will be +// called when block is closed in the inline parsing. +type CloseBlocker interface { + // CloseBlock will be called when a block is closed. + CloseBlock(parent ast.Node, block text.Reader, pc Context) +} + +// A ParagraphTransformer transforms parsed Paragraph nodes. +// For example, link references are searched in parsed Paragraphs. +type ParagraphTransformer interface { + // Transform transforms the given paragraph. + Transform(node *ast.Paragraph, reader text.Reader, pc Context) +} + +// ASTTransformer transforms entire Markdown document AST tree. +type ASTTransformer interface { + // Transform transforms the given AST tree. + Transform(node *ast.Document, reader text.Reader, pc Context) +} + +// DefaultBlockParsers returns a new list of default BlockParsers. +// Priorities of default BlockParsers are: +// +// SetextHeadingParser, 100 +// ThematicBreakParser, 200 +// ListParser, 300 +// ListItemParser, 400 +// CodeBlockParser, 500 +// ATXHeadingParser, 600 +// FencedCodeBlockParser, 700 +// BlockquoteParser, 800 +// HTMLBlockParser, 900 +// ParagraphParser, 1000 +func DefaultBlockParsers() []util.PrioritizedValue { + return []util.PrioritizedValue{ + util.Prioritized(NewSetextHeadingParser(), 100), + util.Prioritized(NewThematicBreakParser(), 200), + util.Prioritized(NewListParser(), 300), + util.Prioritized(NewListItemParser(), 400), + util.Prioritized(NewCodeBlockParser(), 500), + util.Prioritized(NewATXHeadingParser(), 600), + util.Prioritized(NewFencedCodeBlockParser(), 700), + util.Prioritized(NewBlockquoteParser(), 800), + util.Prioritized(NewHTMLBlockParser(), 900), + util.Prioritized(NewParagraphParser(), 1000), + } +} + +// DefaultInlineParsers returns a new list of default InlineParsers. +// Priorities of default InlineParsers are: +// +// CodeSpanParser, 100 +// LinkParser, 200 +// AutoLinkParser, 300 +// RawHTMLParser, 400 +// EmphasisParser, 500 +func DefaultInlineParsers() []util.PrioritizedValue { + return []util.PrioritizedValue{ + util.Prioritized(NewCodeSpanParser(), 100), + util.Prioritized(NewLinkParser(), 200), + util.Prioritized(NewAutoLinkParser(), 300), + util.Prioritized(NewRawHTMLParser(), 400), + util.Prioritized(NewEmphasisParser(), 500), + } +} + +// DefaultParagraphTransformers returns a new list of default ParagraphTransformers. +// Priorities of default ParagraphTransformers are: +// +// LinkReferenceParagraphTransformer, 100 +func DefaultParagraphTransformers() []util.PrioritizedValue { + return []util.PrioritizedValue{ + util.Prioritized(LinkReferenceParagraphTransformer, 100), + } +} + +// A Block struct holds a node and correspond parser pair. +type Block struct { + // Node is a BlockNode. + Node ast.Node + // Parser is a BlockParser. + Parser BlockParser +} + +type parser struct { + options map[OptionName]interface{} + blockParsers [256][]BlockParser + freeBlockParsers []BlockParser + inlineParsers [256][]InlineParser + closeBlockers []CloseBlocker + paragraphTransformers []ParagraphTransformer + astTransformers []ASTTransformer + config *Config + initSync sync.Once +} + +type withBlockParsers struct { + value []util.PrioritizedValue +} + +func (o *withBlockParsers) SetParserOption(c *Config) { + c.BlockParsers = append(c.BlockParsers, o.value...) +} + +// WithBlockParsers is a functional option that allow you to add +// BlockParsers to the parser. +func WithBlockParsers(bs ...util.PrioritizedValue) Option { + return &withBlockParsers{bs} +} + +type withInlineParsers struct { + value []util.PrioritizedValue +} + +func (o *withInlineParsers) SetParserOption(c *Config) { + c.InlineParsers = append(c.InlineParsers, o.value...) +} + +// WithInlineParsers is a functional option that allow you to add +// InlineParsers to the parser. +func WithInlineParsers(bs ...util.PrioritizedValue) Option { + return &withInlineParsers{bs} +} + +type withParagraphTransformers struct { + value []util.PrioritizedValue +} + +func (o *withParagraphTransformers) SetParserOption(c *Config) { + c.ParagraphTransformers = append(c.ParagraphTransformers, o.value...) +} + +// WithParagraphTransformers is a functional option that allow you to add +// ParagraphTransformers to the parser. +func WithParagraphTransformers(ps ...util.PrioritizedValue) Option { + return &withParagraphTransformers{ps} +} + +type withASTTransformers struct { + value []util.PrioritizedValue +} + +func (o *withASTTransformers) SetParserOption(c *Config) { + c.ASTTransformers = append(c.ASTTransformers, o.value...) +} + +// WithASTTransformers is a functional option that allow you to add +// ASTTransformers to the parser. +func WithASTTransformers(ps ...util.PrioritizedValue) Option { + return &withASTTransformers{ps} +} + +type withOption struct { + name OptionName + value interface{} +} + +func (o *withOption) SetParserOption(c *Config) { + c.Options[o.name] = o.value +} + +// WithOption is a functional option that allow you to set +// an arbitrary option to the parser. +func WithOption(name OptionName, value interface{}) Option { + return &withOption{name, value} +} + +// NewParser returns a new Parser with given options. +func NewParser(options ...Option) Parser { + config := NewConfig() + for _, opt := range options { + opt.SetParserOption(config) + } + + p := &parser{ + options: map[OptionName]interface{}{}, + config: config, + } + + return p +} + +func (p *parser) AddOptions(opts ...Option) { + for _, opt := range opts { + opt.SetParserOption(p.config) + } +} + +func (p *parser) addBlockParser(v util.PrioritizedValue, options map[OptionName]interface{}) { + bp, ok := v.Value.(BlockParser) + if !ok { + panic(fmt.Sprintf("%v is not a BlockParser", v.Value)) + } + tcs := bp.Trigger() + so, ok := v.Value.(SetOptioner) + if ok { + for oname, ovalue := range options { + so.SetOption(oname, ovalue) + } + } + if tcs == nil { + p.freeBlockParsers = append(p.freeBlockParsers, bp) + } else { + for _, tc := range tcs { + if p.blockParsers[tc] == nil { + p.blockParsers[tc] = []BlockParser{} + } + p.blockParsers[tc] = append(p.blockParsers[tc], bp) + } + } +} + +func (p *parser) addInlineParser(v util.PrioritizedValue, options map[OptionName]interface{}) { + ip, ok := v.Value.(InlineParser) + if !ok { + panic(fmt.Sprintf("%v is not a InlineParser", v.Value)) + } + tcs := ip.Trigger() + so, ok := v.Value.(SetOptioner) + if ok { + for oname, ovalue := range options { + so.SetOption(oname, ovalue) + } + } + if cb, ok := ip.(CloseBlocker); ok { + p.closeBlockers = append(p.closeBlockers, cb) + } + for _, tc := range tcs { + if p.inlineParsers[tc] == nil { + p.inlineParsers[tc] = []InlineParser{} + } + p.inlineParsers[tc] = append(p.inlineParsers[tc], ip) + } +} + +func (p *parser) addParagraphTransformer(v util.PrioritizedValue, options map[OptionName]interface{}) { + pt, ok := v.Value.(ParagraphTransformer) + if !ok { + panic(fmt.Sprintf("%v is not a ParagraphTransformer", v.Value)) + } + so, ok := v.Value.(SetOptioner) + if ok { + for oname, ovalue := range options { + so.SetOption(oname, ovalue) + } + } + p.paragraphTransformers = append(p.paragraphTransformers, pt) +} + +func (p *parser) addASTTransformer(v util.PrioritizedValue, options map[OptionName]interface{}) { + at, ok := v.Value.(ASTTransformer) + if !ok { + panic(fmt.Sprintf("%v is not a ASTTransformer", v.Value)) + } + so, ok := v.Value.(SetOptioner) + if ok { + for oname, ovalue := range options { + so.SetOption(oname, ovalue) + } + } + p.astTransformers = append(p.astTransformers, at) +} + +// A ParseConfig struct is a data structure that holds configuration of the Parser.Parse. +type ParseConfig struct { + Context Context +} + +// A ParseOption is a functional option type for the Parser.Parse. +type ParseOption func(c *ParseConfig) + +// WithContext is a functional option that allow you to override +// a default context. +func WithContext(context Context) ParseOption { + return func(c *ParseConfig) { + c.Context = context + } +} + +func (p *parser) Parse(reader text.Reader, opts ...ParseOption) ast.Node { + p.initSync.Do(func() { + p.config.BlockParsers.Sort() + for _, v := range p.config.BlockParsers { + p.addBlockParser(v, p.config.Options) + } + for i := range p.blockParsers { + if p.blockParsers[i] != nil { + p.blockParsers[i] = append(p.blockParsers[i], p.freeBlockParsers...) + } + } + + p.config.InlineParsers.Sort() + for _, v := range p.config.InlineParsers { + p.addInlineParser(v, p.config.Options) + } + p.config.ParagraphTransformers.Sort() + for _, v := range p.config.ParagraphTransformers { + p.addParagraphTransformer(v, p.config.Options) + } + p.config.ASTTransformers.Sort() + for _, v := range p.config.ASTTransformers { + p.addASTTransformer(v, p.config.Options) + } + p.config = nil + }) + c := &ParseConfig{} + for _, opt := range opts { + opt(c) + } + if c.Context == nil { + c.Context = NewContext() + } + pc := c.Context + root := ast.NewDocument() + p.parseBlocks(root, reader, pc) + + blockReader := text.NewBlockReader(reader.Source(), nil) + p.walkBlock(root, func(node ast.Node) { + p.parseBlock(blockReader, node, pc) + }) + for _, at := range p.astTransformers { + at.Transform(root, reader, pc) + } + // root.Dump(reader.Source(), 0) + return root +} + +func (p *parser) transformParagraph(node *ast.Paragraph, reader text.Reader, pc Context) bool { + for _, pt := range p.paragraphTransformers { + pt.Transform(node, reader, pc) + if node.Parent() == nil { + return true + } + } + return false +} + +func (p *parser) closeBlocks(from, to int, reader text.Reader, pc Context) { + blocks := pc.OpenedBlocks() + for i := from; i >= to; i-- { + node := blocks[i].Node + blocks[i].Parser.Close(blocks[i].Node, reader, pc) + paragraph, ok := node.(*ast.Paragraph) + if ok && node.Parent() != nil { + p.transformParagraph(paragraph, reader, pc) + } + } + if from == len(blocks)-1 { + blocks = blocks[0:to] + } else { + blocks = append(blocks[0:to], blocks[from+1:]...) + } + pc.SetOpenedBlocks(blocks) +} + +type blockOpenResult int + +const ( + paragraphContinuation blockOpenResult = iota + 1 + newBlocksOpened + noBlocksOpened +) + +func (p *parser) openBlocks(parent ast.Node, blankLine bool, reader text.Reader, pc Context) blockOpenResult { + result := blockOpenResult(noBlocksOpened) + continuable := false + lastBlock := pc.LastOpenedBlock() + if lastBlock.Node != nil { + continuable = ast.IsParagraph(lastBlock.Node) + } +retry: + var bps []BlockParser + line, _ := reader.PeekLine() + w, pos := util.IndentWidth(line, reader.LineOffset()) + if w >= len(line) { + pc.SetBlockOffset(-1) + pc.SetBlockIndent(-1) + } else { + pc.SetBlockOffset(pos) + pc.SetBlockIndent(w) + } + if line == nil || line[0] == '\n' { + goto continuable + } + bps = p.freeBlockParsers + if pos < len(line) { + bps = p.blockParsers[line[pos]] + if bps == nil { + bps = p.freeBlockParsers + } + } + if bps == nil { + goto continuable + } + + for _, bp := range bps { + if continuable && result == noBlocksOpened && !bp.CanInterruptParagraph() { + continue + } + if w > 3 && !bp.CanAcceptIndentedLine() { + continue + } + lastBlock = pc.LastOpenedBlock() + last := lastBlock.Node + node, state := bp.Open(parent, reader, pc) + if node != nil { + // Parser requires last node to be a paragraph. + // With table extension: + // + // 0 + // -: + // - + // + // '-' on 3rd line seems a Setext heading because 1st and 2nd lines + // are being paragraph when the Settext heading parser tries to parse the 3rd + // line. + // But 1st line and 2nd line are a table. Thus this paragraph will be transformed + // by a paragraph transformer. So this text should be converted to a table and + // an empty list. + if state&RequireParagraph != 0 { + if last == parent.LastChild() { + // Opened paragraph may be transformed by ParagraphTransformers in + // closeBlocks(). + lastBlock.Parser.Close(last, reader, pc) + blocks := pc.OpenedBlocks() + pc.SetOpenedBlocks(blocks[0 : len(blocks)-1]) + if p.transformParagraph(last.(*ast.Paragraph), reader, pc) { + // Paragraph has been transformed. + // So this parser is considered as failing. + continuable = false + goto retry + } + } + } + node.SetBlankPreviousLines(blankLine) + if last != nil && last.Parent() == nil { + lastPos := len(pc.OpenedBlocks()) - 1 + p.closeBlocks(lastPos, lastPos, reader, pc) + } + parent.AppendChild(parent, node) + result = newBlocksOpened + be := Block{node, bp} + pc.SetOpenedBlocks(append(pc.OpenedBlocks(), be)) + if state&HasChildren != 0 { + parent = node + goto retry // try child block + } + break // no children, can not open more blocks on this line + } + } + +continuable: + if result == noBlocksOpened && continuable { + state := lastBlock.Parser.Continue(lastBlock.Node, reader, pc) + if state&Continue != 0 { + result = paragraphContinuation + } + } + return result +} + +type lineStat struct { + lineNum int + level int + isBlank bool +} + +func isBlankLine(lineNum, level int, stats []lineStat) bool { + ret := true + for i := len(stats) - 1 - level; i >= 0; i-- { + ret = false + s := stats[i] + if s.lineNum == lineNum { + if s.level < level && s.isBlank { + return true + } else if s.level == level { + return s.isBlank + } + } + if s.lineNum < lineNum { + return ret + } + } + return ret +} + +func (p *parser) parseBlocks(parent ast.Node, reader text.Reader, pc Context) { + pc.SetOpenedBlocks([]Block{}) + blankLines := make([]lineStat, 0, 128) + isBlank := false + for { // process blocks separated by blank lines + _, lines, ok := reader.SkipBlankLines() + if !ok { + return + } + lineNum, _ := reader.Position() + if lines != 0 { + blankLines = blankLines[0:0] + l := len(pc.OpenedBlocks()) + for i := 0; i < l; i++ { + blankLines = append(blankLines, lineStat{lineNum - 1, i, lines != 0}) + } + } + isBlank = isBlankLine(lineNum-1, 0, blankLines) + // first, we try to open blocks + if p.openBlocks(parent, isBlank, reader, pc) != newBlocksOpened { + return + } + reader.AdvanceLine() + for { // process opened blocks line by line + openedBlocks := pc.OpenedBlocks() + l := len(openedBlocks) + if l == 0 { + break + } + lastIndex := l - 1 + for i := 0; i < l; i++ { + be := openedBlocks[i] + line, _ := reader.PeekLine() + if line == nil { + p.closeBlocks(lastIndex, 0, reader, pc) + reader.AdvanceLine() + return + } + lineNum, _ := reader.Position() + blankLines = append(blankLines, lineStat{lineNum, i, util.IsBlank(line)}) + // If node is a paragraph, p.openBlocks determines whether it is continuable. + // So we do not process paragraphs here. + if !ast.IsParagraph(be.Node) { + state := be.Parser.Continue(be.Node, reader, pc) + if state&Continue != 0 { + // When current node is a container block and has no children, + // we try to open new child nodes + if state&HasChildren != 0 && i == lastIndex { + isBlank = isBlankLine(lineNum-1, i, blankLines) + p.openBlocks(be.Node, isBlank, reader, pc) + break + } + continue + } + } + // current node may be closed or lazy continuation + isBlank = isBlankLine(lineNum-1, i, blankLines) + thisParent := parent + if i != 0 { + thisParent = openedBlocks[i-1].Node + } + lastNode := openedBlocks[lastIndex].Node + result := p.openBlocks(thisParent, isBlank, reader, pc) + if result != paragraphContinuation { + // lastNode is a paragraph and was transformed by the paragraph + // transformers. + if openedBlocks[lastIndex].Node != lastNode { + lastIndex-- + } + p.closeBlocks(lastIndex, i, reader, pc) + } + break + } + + reader.AdvanceLine() + } + } +} + +func (p *parser) walkBlock(block ast.Node, cb func(node ast.Node)) { + for c := block.FirstChild(); c != nil; c = c.NextSibling() { + p.walkBlock(c, cb) + } + cb(block) +} + +const ( + lineBreakHard uint8 = 1 << iota + lineBreakSoft + lineBreakVisible +) + +func (p *parser) parseBlock(block text.BlockReader, parent ast.Node, pc Context) { + if parent.IsRaw() { + return + } + escaped := false + source := block.Source() + block.Reset(parent.Lines()) + for { + retry: + line, _ := block.PeekLine() + if line == nil { + break + } + lineLength := len(line) + var lineBreakFlags uint8 = 0 + hasNewLine := line[lineLength-1] == '\n' + if ((lineLength >= 3 && line[lineLength-2] == '\\' && line[lineLength-3] != '\\') || (lineLength == 2 && line[lineLength-2] == '\\')) && hasNewLine { // ends with \\n + lineLength -= 2 + lineBreakFlags |= lineBreakHard | lineBreakVisible + } else if ((lineLength >= 4 && line[lineLength-3] == '\\' && line[lineLength-2] == '\r' && line[lineLength-4] != '\\') || (lineLength == 3 && line[lineLength-3] == '\\' && line[lineLength-2] == '\r')) && hasNewLine { // ends with \\r\n + lineLength -= 3 + lineBreakFlags |= lineBreakHard | lineBreakVisible + } else if lineLength >= 3 && line[lineLength-3] == ' ' && line[lineLength-2] == ' ' && hasNewLine { // ends with [space][space]\n + lineLength -= 3 + lineBreakFlags |= lineBreakHard + } else if lineLength >= 4 && line[lineLength-4] == ' ' && line[lineLength-3] == ' ' && line[lineLength-2] == '\r' && hasNewLine { // ends with [space][space]\r\n + lineLength -= 4 + lineBreakFlags |= lineBreakHard + } else if hasNewLine { + // If the line ends with a newline character, but it is not a hardlineBreak, then it is a softLinebreak + // If the line ends with a hardlineBreak, then it cannot end with a softLinebreak + // See https://spec.commonmark.org/0.30/#soft-line-breaks + lineBreakFlags |= lineBreakSoft + } + + l, startPosition := block.Position() + n := 0 + for i := 0; i < lineLength; i++ { + c := line[i] + if c == '\n' { + break + } + isSpace := util.IsSpace(c) + isPunct := util.IsPunct(c) + if (isPunct && !escaped) || isSpace || i == 0 { + parserChar := c + if isSpace || (i == 0 && !isPunct) { + parserChar = ' ' + } + ips := p.inlineParsers[parserChar] + if ips != nil { + block.Advance(n) + n = 0 + savedLine, savedPosition := block.Position() + if i != 0 { + _, currentPosition := block.Position() + ast.MergeOrAppendTextSegment(parent, startPosition.Between(currentPosition)) + _, startPosition = block.Position() + } + var inlineNode ast.Node + for _, ip := range ips { + inlineNode = ip.Parse(parent, block, pc) + if inlineNode != nil { + break + } + block.SetPosition(savedLine, savedPosition) + } + if inlineNode != nil { + parent.AppendChild(parent, inlineNode) + goto retry + } + } + } + if escaped { + escaped = false + n++ + continue + } + + if c == '\\' { + escaped = true + n++ + continue + } + + escaped = false + n++ + } + if n != 0 { + block.Advance(n) + } + currentL, currentPosition := block.Position() + if l != currentL { + continue + } + diff := startPosition.Between(currentPosition) + var text *ast.Text + if lineBreakFlags&(lineBreakHard|lineBreakVisible) == lineBreakHard|lineBreakVisible { + text = ast.NewTextSegment(diff) + } else { + text = ast.NewTextSegment(diff.TrimRightSpace(source)) + } + text.SetSoftLineBreak(lineBreakFlags&lineBreakSoft != 0) + text.SetHardLineBreak(lineBreakFlags&lineBreakHard != 0) + parent.AppendChild(parent, text) + block.AdvanceLine() + } + + ProcessDelimiters(nil, pc) + for _, ip := range p.closeBlockers { + ip.CloseBlock(parent, block, pc) + } +} diff --git a/vendor/github.com/yuin/goldmark/parser/raw_html.go b/vendor/github.com/yuin/goldmark/parser/raw_html.go new file mode 100644 index 00000000..1fb6c974 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/raw_html.go @@ -0,0 +1,104 @@ +package parser + +import ( + "bytes" + "regexp" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +type rawHTMLParser struct { +} + +var defaultRawHTMLParser = &rawHTMLParser{} + +// NewRawHTMLParser return a new InlineParser that can parse +// inline htmls +func NewRawHTMLParser() InlineParser { + return defaultRawHTMLParser +} + +func (s *rawHTMLParser) Trigger() []byte { + return []byte{'<'} +} + +func (s *rawHTMLParser) Parse(parent ast.Node, block text.Reader, pc Context) ast.Node { + line, _ := block.PeekLine() + if len(line) > 1 && util.IsAlphaNumeric(line[1]) { + return s.parseMultiLineRegexp(openTagRegexp, block, pc) + } + if len(line) > 2 && line[1] == '/' && util.IsAlphaNumeric(line[2]) { + return s.parseMultiLineRegexp(closeTagRegexp, block, pc) + } + if bytes.HasPrefix(line, []byte("<!--")) { + return s.parseMultiLineRegexp(commentRegexp, block, pc) + } + if bytes.HasPrefix(line, []byte("<?")) { + return s.parseSingleLineRegexp(processingInstructionRegexp, block, pc) + } + if len(line) > 2 && line[1] == '!' && line[2] >= 'A' && line[2] <= 'Z' { + return s.parseSingleLineRegexp(declRegexp, block, pc) + } + if bytes.HasPrefix(line, []byte("<![CDATA[")) { + return s.parseMultiLineRegexp(cdataRegexp, block, pc) + } + return nil +} + +var tagnamePattern = `([A-Za-z][A-Za-z0-9-]*)` + +var attributePattern = `(?:[\r\n \t]+[a-zA-Z_:][a-zA-Z0-9:._-]*(?:[\r\n \t]*=[\r\n \t]*(?:[^\"'=<>` + "`" + `\x00-\x20]+|'[^']*'|"[^"]*"))?)` +var openTagRegexp = regexp.MustCompile("^<" + tagnamePattern + attributePattern + `*[ \t]*/?>`) +var closeTagRegexp = regexp.MustCompile("^</" + tagnamePattern + `\s*>`) +var commentRegexp = regexp.MustCompile(`^<!---->|<!--(?:-?[^>-])(?:-?[^-])*-->`) +var processingInstructionRegexp = regexp.MustCompile(`^(?:<\?).*?(?:\?>)`) +var declRegexp = regexp.MustCompile(`^<![A-Z]+\s+[^>]*>`) +var cdataRegexp = regexp.MustCompile(`<!\[CDATA\[[\s\S]*?\]\]>`) + +func (s *rawHTMLParser) parseSingleLineRegexp(reg *regexp.Regexp, block text.Reader, pc Context) ast.Node { + line, segment := block.PeekLine() + match := reg.FindSubmatchIndex(line) + if match == nil { + return nil + } + node := ast.NewRawHTML() + node.Segments.Append(segment.WithStop(segment.Start + match[1])) + block.Advance(match[1]) + return node +} + +func (s *rawHTMLParser) parseMultiLineRegexp(reg *regexp.Regexp, block text.Reader, pc Context) ast.Node { + sline, ssegment := block.Position() + if block.Match(reg) { + node := ast.NewRawHTML() + eline, esegment := block.Position() + block.SetPosition(sline, ssegment) + for { + line, segment := block.PeekLine() + if line == nil { + break + } + l, _ := block.Position() + start := segment.Start + if l == sline { + start = ssegment.Start + } + end := segment.Stop + if l == eline { + end = esegment.Start + } + + node.Segments.Append(text.NewSegment(start, end)) + if l == eline { + block.Advance(end - start) + break + } else { + block.AdvanceLine() + } + } + return node + } + return nil +} diff --git a/vendor/github.com/yuin/goldmark/parser/setext_headings.go b/vendor/github.com/yuin/goldmark/parser/setext_headings.go new file mode 100644 index 00000000..686efe17 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/setext_headings.go @@ -0,0 +1,126 @@ +package parser + +import ( + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +var temporaryParagraphKey = NewContextKey() + +type setextHeadingParser struct { + HeadingConfig +} + +func matchesSetextHeadingBar(line []byte) (byte, bool) { + start := 0 + end := len(line) + space := util.TrimLeftLength(line, []byte{' '}) + if space > 3 { + return 0, false + } + start += space + level1 := util.TrimLeftLength(line[start:end], []byte{'='}) + c := byte('=') + var level2 int + if level1 == 0 { + level2 = util.TrimLeftLength(line[start:end], []byte{'-'}) + c = '-' + } + if util.IsSpace(line[end-1]) { + end -= util.TrimRightSpaceLength(line[start:end]) + } + if !((level1 > 0 && start+level1 == end) || (level2 > 0 && start+level2 == end)) { + return 0, false + } + return c, true +} + +// NewSetextHeadingParser return a new BlockParser that can parse Setext headings. +func NewSetextHeadingParser(opts ...HeadingOption) BlockParser { + p := &setextHeadingParser{} + for _, o := range opts { + o.SetHeadingOption(&p.HeadingConfig) + } + return p +} + +func (b *setextHeadingParser) Trigger() []byte { + return []byte{'-', '='} +} + +func (b *setextHeadingParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) { + last := pc.LastOpenedBlock().Node + if last == nil { + return nil, NoChildren + } + paragraph, ok := last.(*ast.Paragraph) + if !ok || paragraph.Parent() != parent { + return nil, NoChildren + } + line, segment := reader.PeekLine() + c, ok := matchesSetextHeadingBar(line) + if !ok { + return nil, NoChildren + } + level := 1 + if c == '-' { + level = 2 + } + node := ast.NewHeading(level) + node.Lines().Append(segment) + pc.Set(temporaryParagraphKey, last) + return node, NoChildren | RequireParagraph +} + +func (b *setextHeadingParser) Continue(node ast.Node, reader text.Reader, pc Context) State { + return Close +} + +func (b *setextHeadingParser) Close(node ast.Node, reader text.Reader, pc Context) { + heading := node.(*ast.Heading) + segment := node.Lines().At(0) + heading.Lines().Clear() + tmp := pc.Get(temporaryParagraphKey).(*ast.Paragraph) + pc.Set(temporaryParagraphKey, nil) + if tmp.Lines().Len() == 0 { + next := heading.NextSibling() + segment = segment.TrimLeftSpace(reader.Source()) + if next == nil || !ast.IsParagraph(next) { + para := ast.NewParagraph() + para.Lines().Append(segment) + heading.Parent().InsertAfter(heading.Parent(), heading, para) + } else { + next.(ast.Node).Lines().Unshift(segment) + } + heading.Parent().RemoveChild(heading.Parent(), heading) + } else { + heading.SetLines(tmp.Lines()) + heading.SetBlankPreviousLines(tmp.HasBlankPreviousLines()) + tp := tmp.Parent() + if tp != nil { + tp.RemoveChild(tp, tmp) + } + } + + if b.Attribute { + parseLastLineAttributes(node, reader, pc) + } + + if b.AutoHeadingID { + id, ok := node.AttributeString("id") + if !ok { + generateAutoHeadingID(heading, reader, pc) + } else { + pc.IDs().Put(id.([]byte)) + } + } +} + +func (b *setextHeadingParser) CanInterruptParagraph() bool { + return true +} + +func (b *setextHeadingParser) CanAcceptIndentedLine() bool { + return false +} diff --git a/vendor/github.com/yuin/goldmark/parser/thematic_break.go b/vendor/github.com/yuin/goldmark/parser/thematic_break.go new file mode 100644 index 00000000..db20a1e7 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/parser/thematic_break.go @@ -0,0 +1,75 @@ +package parser + +import ( + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +type thematicBreakPraser struct { +} + +var defaultThematicBreakPraser = &thematicBreakPraser{} + +// NewThematicBreakParser returns a new BlockParser that +// parses thematic breaks. +func NewThematicBreakParser() BlockParser { + return defaultThematicBreakPraser +} + +func isThematicBreak(line []byte, offset int) bool { + w, pos := util.IndentWidth(line, offset) + if w > 3 { + return false + } + mark := byte(0) + count := 0 + for i := pos; i < len(line); i++ { + c := line[i] + if util.IsSpace(c) { + continue + } + if mark == 0 { + mark = c + count = 1 + if mark == '*' || mark == '-' || mark == '_' { + continue + } + return false + } + if c != mark { + return false + } + count++ + } + return count > 2 +} + +func (b *thematicBreakPraser) Trigger() []byte { + return []byte{'-', '*', '_'} +} + +func (b *thematicBreakPraser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) { + line, segment := reader.PeekLine() + if isThematicBreak(line, reader.LineOffset()) { + reader.Advance(segment.Len() - 1) + return ast.NewThematicBreak(), NoChildren + } + return nil, NoChildren +} + +func (b *thematicBreakPraser) Continue(node ast.Node, reader text.Reader, pc Context) State { + return Close +} + +func (b *thematicBreakPraser) Close(node ast.Node, reader text.Reader, pc Context) { + // nothing to do +} + +func (b *thematicBreakPraser) CanInterruptParagraph() bool { + return true +} + +func (b *thematicBreakPraser) CanAcceptIndentedLine() bool { + return false +} diff --git a/vendor/github.com/yuin/goldmark/renderer/html/html.go b/vendor/github.com/yuin/goldmark/renderer/html/html.go new file mode 100644 index 00000000..7f5b9fed --- /dev/null +++ b/vendor/github.com/yuin/goldmark/renderer/html/html.go @@ -0,0 +1,831 @@ +package html + +import ( + "bytes" + "fmt" + "strconv" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/util" +) + +// A Config struct has configurations for the HTML based renderers. +type Config struct { + Writer Writer + HardWraps bool + XHTML bool + Unsafe bool +} + +// NewConfig returns a new Config with defaults. +func NewConfig() Config { + return Config{ + Writer: DefaultWriter, + HardWraps: false, + XHTML: false, + Unsafe: false, + } +} + +// SetOption implements renderer.NodeRenderer.SetOption. +func (c *Config) SetOption(name renderer.OptionName, value interface{}) { + switch name { + case optHardWraps: + c.HardWraps = value.(bool) + case optXHTML: + c.XHTML = value.(bool) + case optUnsafe: + c.Unsafe = value.(bool) + case optTextWriter: + c.Writer = value.(Writer) + } +} + +// An Option interface sets options for HTML based renderers. +type Option interface { + SetHTMLOption(*Config) +} + +// TextWriter is an option name used in WithWriter. +const optTextWriter renderer.OptionName = "Writer" + +type withWriter struct { + value Writer +} + +func (o *withWriter) SetConfig(c *renderer.Config) { + c.Options[optTextWriter] = o.value +} + +func (o *withWriter) SetHTMLOption(c *Config) { + c.Writer = o.value +} + +// WithWriter is a functional option that allow you to set the given writer to +// the renderer. +func WithWriter(writer Writer) interface { + renderer.Option + Option +} { + return &withWriter{writer} +} + +// HardWraps is an option name used in WithHardWraps. +const optHardWraps renderer.OptionName = "HardWraps" + +type withHardWraps struct { +} + +func (o *withHardWraps) SetConfig(c *renderer.Config) { + c.Options[optHardWraps] = true +} + +func (o *withHardWraps) SetHTMLOption(c *Config) { + c.HardWraps = true +} + +// WithHardWraps is a functional option that indicates whether softline breaks +// should be rendered as '<br>'. +func WithHardWraps() interface { + renderer.Option + Option +} { + return &withHardWraps{} +} + +// XHTML is an option name used in WithXHTML. +const optXHTML renderer.OptionName = "XHTML" + +type withXHTML struct { +} + +func (o *withXHTML) SetConfig(c *renderer.Config) { + c.Options[optXHTML] = true +} + +func (o *withXHTML) SetHTMLOption(c *Config) { + c.XHTML = true +} + +// WithXHTML is a functional option indicates that nodes should be rendered in +// xhtml instead of HTML5. +func WithXHTML() interface { + Option + renderer.Option +} { + return &withXHTML{} +} + +// Unsafe is an option name used in WithUnsafe. +const optUnsafe renderer.OptionName = "Unsafe" + +type withUnsafe struct { +} + +func (o *withUnsafe) SetConfig(c *renderer.Config) { + c.Options[optUnsafe] = true +} + +func (o *withUnsafe) SetHTMLOption(c *Config) { + c.Unsafe = true +} + +// WithUnsafe is a functional option that renders dangerous contents +// (raw htmls and potentially dangerous links) as it is. +func WithUnsafe() interface { + renderer.Option + Option +} { + return &withUnsafe{} +} + +// A Renderer struct is an implementation of renderer.NodeRenderer that renders +// nodes as (X)HTML. +type Renderer struct { + Config +} + +// NewRenderer returns a new Renderer with given options. +func NewRenderer(opts ...Option) renderer.NodeRenderer { + r := &Renderer{ + Config: NewConfig(), + } + + for _, opt := range opts { + opt.SetHTMLOption(&r.Config) + } + return r +} + +// RegisterFuncs implements NodeRenderer.RegisterFuncs . +func (r *Renderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { + // blocks + + reg.Register(ast.KindDocument, r.renderDocument) + reg.Register(ast.KindHeading, r.renderHeading) + reg.Register(ast.KindBlockquote, r.renderBlockquote) + reg.Register(ast.KindCodeBlock, r.renderCodeBlock) + reg.Register(ast.KindFencedCodeBlock, r.renderFencedCodeBlock) + reg.Register(ast.KindHTMLBlock, r.renderHTMLBlock) + reg.Register(ast.KindList, r.renderList) + reg.Register(ast.KindListItem, r.renderListItem) + reg.Register(ast.KindParagraph, r.renderParagraph) + reg.Register(ast.KindTextBlock, r.renderTextBlock) + reg.Register(ast.KindThematicBreak, r.renderThematicBreak) + + // inlines + + reg.Register(ast.KindAutoLink, r.renderAutoLink) + reg.Register(ast.KindCodeSpan, r.renderCodeSpan) + reg.Register(ast.KindEmphasis, r.renderEmphasis) + reg.Register(ast.KindImage, r.renderImage) + reg.Register(ast.KindLink, r.renderLink) + reg.Register(ast.KindRawHTML, r.renderRawHTML) + reg.Register(ast.KindText, r.renderText) + reg.Register(ast.KindString, r.renderString) +} + +func (r *Renderer) writeLines(w util.BufWriter, source []byte, n ast.Node) { + l := n.Lines().Len() + for i := 0; i < l; i++ { + line := n.Lines().At(i) + r.Writer.RawWrite(w, line.Value(source)) + } +} + +// GlobalAttributeFilter defines attribute names which any elements can have. +var GlobalAttributeFilter = util.NewBytesFilter( + []byte("accesskey"), + []byte("autocapitalize"), + []byte("class"), + []byte("contenteditable"), + []byte("contextmenu"), + []byte("dir"), + []byte("draggable"), + []byte("dropzone"), + []byte("hidden"), + []byte("id"), + []byte("itemprop"), + []byte("lang"), + []byte("slot"), + []byte("spellcheck"), + []byte("style"), + []byte("tabindex"), + []byte("title"), + []byte("translate"), +) + +func (r *Renderer) renderDocument(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + // nothing to do + return ast.WalkContinue, nil +} + +// HeadingAttributeFilter defines attribute names which heading elements can have +var HeadingAttributeFilter = GlobalAttributeFilter + +func (r *Renderer) renderHeading(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.Heading) + if entering { + _, _ = w.WriteString("<h") + _ = w.WriteByte("0123456"[n.Level]) + if n.Attributes() != nil { + RenderAttributes(w, node, HeadingAttributeFilter) + } + _ = w.WriteByte('>') + } else { + _, _ = w.WriteString("</h") + _ = w.WriteByte("0123456"[n.Level]) + _, _ = w.WriteString(">\n") + } + return ast.WalkContinue, nil +} + +// BlockquoteAttributeFilter defines attribute names which blockquote elements can have +var BlockquoteAttributeFilter = GlobalAttributeFilter.Extend( + []byte("cite"), +) + +func (r *Renderer) renderBlockquote(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + if n.Attributes() != nil { + _, _ = w.WriteString("<blockquote") + RenderAttributes(w, n, BlockquoteAttributeFilter) + _ = w.WriteByte('>') + } else { + _, _ = w.WriteString("<blockquote>\n") + } + } else { + _, _ = w.WriteString("</blockquote>\n") + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderCodeBlock(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + _, _ = w.WriteString("<pre><code>") + r.writeLines(w, source, n) + } else { + _, _ = w.WriteString("</code></pre>\n") + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderFencedCodeBlock(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.FencedCodeBlock) + if entering { + _, _ = w.WriteString("<pre><code") + language := n.Language(source) + if language != nil { + _, _ = w.WriteString(" class=\"language-") + r.Writer.Write(w, language) + _, _ = w.WriteString("\"") + } + _ = w.WriteByte('>') + r.writeLines(w, source, n) + } else { + _, _ = w.WriteString("</code></pre>\n") + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderHTMLBlock(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.HTMLBlock) + if entering { + if r.Unsafe { + l := n.Lines().Len() + for i := 0; i < l; i++ { + line := n.Lines().At(i) + r.Writer.SecureWrite(w, line.Value(source)) + } + } else { + _, _ = w.WriteString("<!-- raw HTML omitted -->\n") + } + } else { + if n.HasClosure() { + if r.Unsafe { + closure := n.ClosureLine + r.Writer.SecureWrite(w, closure.Value(source)) + } else { + _, _ = w.WriteString("<!-- raw HTML omitted -->\n") + } + } + } + return ast.WalkContinue, nil +} + +// ListAttributeFilter defines attribute names which list elements can have. +var ListAttributeFilter = GlobalAttributeFilter.Extend( + []byte("start"), + []byte("reversed"), + []byte("type"), +) + +func (r *Renderer) renderList(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.List) + tag := "ul" + if n.IsOrdered() { + tag = "ol" + } + if entering { + _ = w.WriteByte('<') + _, _ = w.WriteString(tag) + if n.IsOrdered() && n.Start != 1 { + fmt.Fprintf(w, " start=\"%d\"", n.Start) + } + if n.Attributes() != nil { + RenderAttributes(w, n, ListAttributeFilter) + } + _, _ = w.WriteString(">\n") + } else { + _, _ = w.WriteString("</") + _, _ = w.WriteString(tag) + _, _ = w.WriteString(">\n") + } + return ast.WalkContinue, nil +} + +// ListItemAttributeFilter defines attribute names which list item elements can have. +var ListItemAttributeFilter = GlobalAttributeFilter.Extend( + []byte("value"), +) + +func (r *Renderer) renderListItem(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + if n.Attributes() != nil { + _, _ = w.WriteString("<li") + RenderAttributes(w, n, ListItemAttributeFilter) + _ = w.WriteByte('>') + } else { + _, _ = w.WriteString("<li>") + } + fc := n.FirstChild() + if fc != nil { + if _, ok := fc.(*ast.TextBlock); !ok { + _ = w.WriteByte('\n') + } + } + } else { + _, _ = w.WriteString("</li>\n") + } + return ast.WalkContinue, nil +} + +// ParagraphAttributeFilter defines attribute names which paragraph elements can have. +var ParagraphAttributeFilter = GlobalAttributeFilter + +func (r *Renderer) renderParagraph(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + if n.Attributes() != nil { + _, _ = w.WriteString("<p") + RenderAttributes(w, n, ParagraphAttributeFilter) + _ = w.WriteByte('>') + } else { + _, _ = w.WriteString("<p>") + } + } else { + _, _ = w.WriteString("</p>\n") + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderTextBlock(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + if _, ok := n.NextSibling().(ast.Node); ok && n.FirstChild() != nil { + _ = w.WriteByte('\n') + } + } + return ast.WalkContinue, nil +} + +// ThematicAttributeFilter defines attribute names which hr elements can have. +var ThematicAttributeFilter = GlobalAttributeFilter.Extend( + []byte("align"), // [Deprecated] + []byte("color"), // [Not Standardized] + []byte("noshade"), // [Deprecated] + []byte("size"), // [Deprecated] + []byte("width"), // [Deprecated] +) + +func (r *Renderer) renderThematicBreak(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + _, _ = w.WriteString("<hr") + if n.Attributes() != nil { + RenderAttributes(w, n, ThematicAttributeFilter) + } + if r.XHTML { + _, _ = w.WriteString(" />\n") + } else { + _, _ = w.WriteString(">\n") + } + return ast.WalkContinue, nil +} + +// LinkAttributeFilter defines attribute names which link elements can have. +var LinkAttributeFilter = GlobalAttributeFilter.Extend( + []byte("download"), + // []byte("href"), + []byte("hreflang"), + []byte("media"), + []byte("ping"), + []byte("referrerpolicy"), + []byte("rel"), + []byte("shape"), + []byte("target"), +) + +func (r *Renderer) renderAutoLink(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.AutoLink) + if !entering { + return ast.WalkContinue, nil + } + _, _ = w.WriteString(`<a href="`) + url := n.URL(source) + label := n.Label(source) + if n.AutoLinkType == ast.AutoLinkEmail && !bytes.HasPrefix(bytes.ToLower(url), []byte("mailto:")) { + _, _ = w.WriteString("mailto:") + } + _, _ = w.Write(util.EscapeHTML(util.URLEscape(url, false))) + if n.Attributes() != nil { + _ = w.WriteByte('"') + RenderAttributes(w, n, LinkAttributeFilter) + _ = w.WriteByte('>') + } else { + _, _ = w.WriteString(`">`) + } + _, _ = w.Write(util.EscapeHTML(label)) + _, _ = w.WriteString(`</a>`) + return ast.WalkContinue, nil +} + +// CodeAttributeFilter defines attribute names which code elements can have. +var CodeAttributeFilter = GlobalAttributeFilter + +func (r *Renderer) renderCodeSpan(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering { + if n.Attributes() != nil { + _, _ = w.WriteString("<code") + RenderAttributes(w, n, CodeAttributeFilter) + _ = w.WriteByte('>') + } else { + _, _ = w.WriteString("<code>") + } + for c := n.FirstChild(); c != nil; c = c.NextSibling() { + segment := c.(*ast.Text).Segment + value := segment.Value(source) + if bytes.HasSuffix(value, []byte("\n")) { + r.Writer.RawWrite(w, value[:len(value)-1]) + r.Writer.RawWrite(w, []byte(" ")) + } else { + r.Writer.RawWrite(w, value) + } + } + return ast.WalkSkipChildren, nil + } + _, _ = w.WriteString("</code>") + return ast.WalkContinue, nil +} + +// EmphasisAttributeFilter defines attribute names which emphasis elements can have. +var EmphasisAttributeFilter = GlobalAttributeFilter + +func (r *Renderer) renderEmphasis(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.Emphasis) + tag := "em" + if n.Level == 2 { + tag = "strong" + } + if entering { + _ = w.WriteByte('<') + _, _ = w.WriteString(tag) + if n.Attributes() != nil { + RenderAttributes(w, n, EmphasisAttributeFilter) + } + _ = w.WriteByte('>') + } else { + _, _ = w.WriteString("</") + _, _ = w.WriteString(tag) + _ = w.WriteByte('>') + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderLink(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + n := node.(*ast.Link) + if entering { + _, _ = w.WriteString("<a href=\"") + if r.Unsafe || !IsDangerousURL(n.Destination) { + _, _ = w.Write(util.EscapeHTML(util.URLEscape(n.Destination, true))) + } + _ = w.WriteByte('"') + if n.Title != nil { + _, _ = w.WriteString(` title="`) + r.Writer.Write(w, n.Title) + _ = w.WriteByte('"') + } + if n.Attributes() != nil { + RenderAttributes(w, n, LinkAttributeFilter) + } + _ = w.WriteByte('>') + } else { + _, _ = w.WriteString("</a>") + } + return ast.WalkContinue, nil +} + +// ImageAttributeFilter defines attribute names which image elements can have. +var ImageAttributeFilter = GlobalAttributeFilter.Extend( + []byte("align"), + []byte("border"), + []byte("crossorigin"), + []byte("decoding"), + []byte("height"), + []byte("importance"), + []byte("intrinsicsize"), + []byte("ismap"), + []byte("loading"), + []byte("referrerpolicy"), + []byte("sizes"), + []byte("srcset"), + []byte("usemap"), + []byte("width"), +) + +func (r *Renderer) renderImage(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + n := node.(*ast.Image) + _, _ = w.WriteString("<img src=\"") + if r.Unsafe || !IsDangerousURL(n.Destination) { + _, _ = w.Write(util.EscapeHTML(util.URLEscape(n.Destination, true))) + } + _, _ = w.WriteString(`" alt="`) + _, _ = w.Write(util.EscapeHTML(n.Text(source))) + _ = w.WriteByte('"') + if n.Title != nil { + _, _ = w.WriteString(` title="`) + r.Writer.Write(w, n.Title) + _ = w.WriteByte('"') + } + if n.Attributes() != nil { + RenderAttributes(w, n, ImageAttributeFilter) + } + if r.XHTML { + _, _ = w.WriteString(" />") + } else { + _, _ = w.WriteString(">") + } + return ast.WalkSkipChildren, nil +} + +func (r *Renderer) renderRawHTML(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkSkipChildren, nil + } + if r.Unsafe { + n := node.(*ast.RawHTML) + l := n.Segments.Len() + for i := 0; i < l; i++ { + segment := n.Segments.At(i) + _, _ = w.Write(segment.Value(source)) + } + return ast.WalkSkipChildren, nil + } + _, _ = w.WriteString("<!-- raw HTML omitted -->") + return ast.WalkSkipChildren, nil +} + +func (r *Renderer) renderText(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + n := node.(*ast.Text) + segment := n.Segment + if n.IsRaw() { + r.Writer.RawWrite(w, segment.Value(source)) + } else { + r.Writer.Write(w, segment.Value(source)) + if n.HardLineBreak() || (n.SoftLineBreak() && r.HardWraps) { + if r.XHTML { + _, _ = w.WriteString("<br />\n") + } else { + _, _ = w.WriteString("<br>\n") + } + } else if n.SoftLineBreak() { + _ = w.WriteByte('\n') + } + } + return ast.WalkContinue, nil +} + +func (r *Renderer) renderString(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + n := node.(*ast.String) + if n.IsCode() { + _, _ = w.Write(n.Value) + } else { + if n.IsRaw() { + r.Writer.RawWrite(w, n.Value) + } else { + r.Writer.Write(w, n.Value) + } + } + return ast.WalkContinue, nil +} + +var dataPrefix = []byte("data-") + +// RenderAttributes renders given node's attributes. +// You can specify attribute names to render by the filter. +// If filter is nil, RenderAttributes renders all attributes. +func RenderAttributes(w util.BufWriter, node ast.Node, filter util.BytesFilter) { + for _, attr := range node.Attributes() { + if filter != nil && !filter.Contains(attr.Name) { + if !bytes.HasPrefix(attr.Name, dataPrefix) { + continue + } + } + _, _ = w.WriteString(" ") + _, _ = w.Write(attr.Name) + _, _ = w.WriteString(`="`) + // TODO: convert numeric values to strings + _, _ = w.Write(util.EscapeHTML(attr.Value.([]byte))) + _ = w.WriteByte('"') + } +} + +// A Writer interface writes textual contents to a writer. +type Writer interface { + // Write writes the given source to writer with resolving references and unescaping + // backslash escaped characters. + Write(writer util.BufWriter, source []byte) + + // RawWrite writes the given source to writer without resolving references and + // unescaping backslash escaped characters. + RawWrite(writer util.BufWriter, source []byte) + + // SecureWrite writes the given source to writer with replacing insecure characters. + SecureWrite(writer util.BufWriter, source []byte) +} + +var replacementCharacter = []byte("\ufffd") + +type defaultWriter struct { +} + +func escapeRune(writer util.BufWriter, r rune) { + if r < 256 { + v := util.EscapeHTMLByte(byte(r)) + if v != nil { + _, _ = writer.Write(v) + return + } + } + _, _ = writer.WriteRune(util.ToValidRune(r)) +} + +func (d *defaultWriter) SecureWrite(writer util.BufWriter, source []byte) { + n := 0 + l := len(source) + for i := 0; i < l; i++ { + if source[i] == '\u0000' { + _, _ = writer.Write(source[i-n : i]) + n = 0 + _, _ = writer.Write(replacementCharacter) + continue + } + n++ + } + if n != 0 { + _, _ = writer.Write(source[l-n:]) + } +} + +func (d *defaultWriter) RawWrite(writer util.BufWriter, source []byte) { + n := 0 + l := len(source) + for i := 0; i < l; i++ { + v := util.EscapeHTMLByte(source[i]) + if v != nil { + _, _ = writer.Write(source[i-n : i]) + n = 0 + _, _ = writer.Write(v) + continue + } + n++ + } + if n != 0 { + _, _ = writer.Write(source[l-n:]) + } +} + +func (d *defaultWriter) Write(writer util.BufWriter, source []byte) { + escaped := false + var ok bool + limit := len(source) + n := 0 + for i := 0; i < limit; i++ { + c := source[i] + if escaped { + if util.IsPunct(c) { + d.RawWrite(writer, source[n:i-1]) + n = i + escaped = false + continue + } + } + if c == '\x00' { + d.RawWrite(writer, source[n:i]) + d.RawWrite(writer, replacementCharacter) + n = i + 1 + continue + } + if c == '&' { + pos := i + next := i + 1 + if next < limit && source[next] == '#' { + nnext := next + 1 + if nnext < limit { + nc := source[nnext] + // code point like #x22; + if nnext < limit && nc == 'x' || nc == 'X' { + start := nnext + 1 + i, ok = util.ReadWhile(source, [2]int{start, limit}, util.IsHexDecimal) + if ok && i < limit && source[i] == ';' && i-start < 7 { + v, _ := strconv.ParseUint(util.BytesToReadOnlyString(source[start:i]), 16, 32) + d.RawWrite(writer, source[n:pos]) + n = i + 1 + escapeRune(writer, rune(v)) + continue + } + // code point like #1234; + } else if nc >= '0' && nc <= '9' { + start := nnext + i, ok = util.ReadWhile(source, [2]int{start, limit}, util.IsNumeric) + if ok && i < limit && i-start < 8 && source[i] == ';' { + v, _ := strconv.ParseUint(util.BytesToReadOnlyString(source[start:i]), 10, 32) + d.RawWrite(writer, source[n:pos]) + n = i + 1 + escapeRune(writer, rune(v)) + continue + } + } + } + } else { + start := next + i, ok = util.ReadWhile(source, [2]int{start, limit}, util.IsAlphaNumeric) + // entity reference + if ok && i < limit && source[i] == ';' { + name := util.BytesToReadOnlyString(source[start:i]) + entity, ok := util.LookUpHTML5EntityByName(name) + if ok { + d.RawWrite(writer, source[n:pos]) + n = i + 1 + d.RawWrite(writer, entity.Characters) + continue + } + } + } + i = next - 1 + } + if c == '\\' { + escaped = true + continue + } + escaped = false + } + d.RawWrite(writer, source[n:]) +} + +// DefaultWriter is a default implementation of the Writer. +var DefaultWriter = &defaultWriter{} + +var bDataImage = []byte("data:image/") +var bPng = []byte("png;") +var bGif = []byte("gif;") +var bJpeg = []byte("jpeg;") +var bWebp = []byte("webp;") +var bJs = []byte("javascript:") +var bVb = []byte("vbscript:") +var bFile = []byte("file:") +var bData = []byte("data:") + +// IsDangerousURL returns true if the given url seems a potentially dangerous url, +// otherwise false. +func IsDangerousURL(url []byte) bool { + if bytes.HasPrefix(url, bDataImage) && len(url) >= 11 { + v := url[11:] + if bytes.HasPrefix(v, bPng) || bytes.HasPrefix(v, bGif) || + bytes.HasPrefix(v, bJpeg) || bytes.HasPrefix(v, bWebp) { + return false + } + return true + } + return bytes.HasPrefix(url, bJs) || bytes.HasPrefix(url, bVb) || + bytes.HasPrefix(url, bFile) || bytes.HasPrefix(url, bData) +} diff --git a/vendor/github.com/yuin/goldmark/renderer/renderer.go b/vendor/github.com/yuin/goldmark/renderer/renderer.go new file mode 100644 index 00000000..10f6d401 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/renderer/renderer.go @@ -0,0 +1,174 @@ +// Package renderer renders the given AST to certain formats. +package renderer + +import ( + "bufio" + "io" + "sync" + + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/util" +) + +// A Config struct is a data structure that holds configuration of the Renderer. +type Config struct { + Options map[OptionName]interface{} + NodeRenderers util.PrioritizedSlice +} + +// NewConfig returns a new Config +func NewConfig() *Config { + return &Config{ + Options: map[OptionName]interface{}{}, + NodeRenderers: util.PrioritizedSlice{}, + } +} + +// An OptionName is a name of the option. +type OptionName string + +// An Option interface is a functional option type for the Renderer. +type Option interface { + SetConfig(*Config) +} + +type withNodeRenderers struct { + value []util.PrioritizedValue +} + +func (o *withNodeRenderers) SetConfig(c *Config) { + c.NodeRenderers = append(c.NodeRenderers, o.value...) +} + +// WithNodeRenderers is a functional option that allow you to add +// NodeRenderers to the renderer. +func WithNodeRenderers(ps ...util.PrioritizedValue) Option { + return &withNodeRenderers{ps} +} + +type withOption struct { + name OptionName + value interface{} +} + +func (o *withOption) SetConfig(c *Config) { + c.Options[o.name] = o.value +} + +// WithOption is a functional option that allow you to set +// an arbitrary option to the parser. +func WithOption(name OptionName, value interface{}) Option { + return &withOption{name, value} +} + +// A SetOptioner interface sets given option to the object. +type SetOptioner interface { + // SetOption sets given option to the object. + // Unacceptable options may be passed. + // Thus implementations must ignore unacceptable options. + SetOption(name OptionName, value interface{}) +} + +// NodeRendererFunc is a function that renders a given node. +type NodeRendererFunc func(writer util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) + +// A NodeRenderer interface offers NodeRendererFuncs. +type NodeRenderer interface { + // RendererFuncs registers NodeRendererFuncs to given NodeRendererFuncRegisterer. + RegisterFuncs(NodeRendererFuncRegisterer) +} + +// A NodeRendererFuncRegisterer registers +type NodeRendererFuncRegisterer interface { + // Register registers given NodeRendererFunc to this object. + Register(ast.NodeKind, NodeRendererFunc) +} + +// A Renderer interface renders given AST node to given +// writer with given Renderer. +type Renderer interface { + Render(w io.Writer, source []byte, n ast.Node) error + + // AddOptions adds given option to this renderer. + AddOptions(...Option) +} + +type renderer struct { + config *Config + options map[OptionName]interface{} + nodeRendererFuncsTmp map[ast.NodeKind]NodeRendererFunc + maxKind int + nodeRendererFuncs []NodeRendererFunc + initSync sync.Once +} + +// NewRenderer returns a new Renderer with given options. +func NewRenderer(options ...Option) Renderer { + config := NewConfig() + for _, opt := range options { + opt.SetConfig(config) + } + + r := &renderer{ + options: map[OptionName]interface{}{}, + config: config, + nodeRendererFuncsTmp: map[ast.NodeKind]NodeRendererFunc{}, + } + + return r +} + +func (r *renderer) AddOptions(opts ...Option) { + for _, opt := range opts { + opt.SetConfig(r.config) + } +} + +func (r *renderer) Register(kind ast.NodeKind, v NodeRendererFunc) { + r.nodeRendererFuncsTmp[kind] = v + if int(kind) > r.maxKind { + r.maxKind = int(kind) + } +} + +// Render renders the given AST node to the given writer with the given Renderer. +func (r *renderer) Render(w io.Writer, source []byte, n ast.Node) error { + r.initSync.Do(func() { + r.options = r.config.Options + r.config.NodeRenderers.Sort() + l := len(r.config.NodeRenderers) + for i := l - 1; i >= 0; i-- { + v := r.config.NodeRenderers[i] + nr, _ := v.Value.(NodeRenderer) + if se, ok := v.Value.(SetOptioner); ok { + for oname, ovalue := range r.options { + se.SetOption(oname, ovalue) + } + } + nr.RegisterFuncs(r) + } + r.nodeRendererFuncs = make([]NodeRendererFunc, r.maxKind+1) + for kind, nr := range r.nodeRendererFuncsTmp { + r.nodeRendererFuncs[kind] = nr + } + r.config = nil + r.nodeRendererFuncsTmp = nil + }) + writer, ok := w.(util.BufWriter) + if !ok { + writer = bufio.NewWriter(w) + } + err := ast.Walk(n, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + s := ast.WalkStatus(ast.WalkContinue) + var err error + f := r.nodeRendererFuncs[n.Kind()] + if f != nil { + s, err = f(writer, source, n, entering) + } + return s, err + }) + if err != nil { + return err + } + return writer.Flush() +} diff --git a/vendor/github.com/yuin/goldmark/text/reader.go b/vendor/github.com/yuin/goldmark/text/reader.go new file mode 100644 index 00000000..319f1c8b --- /dev/null +++ b/vendor/github.com/yuin/goldmark/text/reader.go @@ -0,0 +1,653 @@ +package text + +import ( + "io" + "regexp" + "unicode/utf8" + + "github.com/yuin/goldmark/util" +) + +const invalidValue = -1 + +// EOF indicates the end of file. +const EOF = byte(0xff) + +// A Reader interface provides abstracted method for reading text. +type Reader interface { + io.RuneReader + + // Source returns a source of the reader. + Source() []byte + + // ResetPosition resets positions. + ResetPosition() + + // Peek returns a byte at current position without advancing the internal pointer. + Peek() byte + + // PeekLine returns the current line without advancing the internal pointer. + PeekLine() ([]byte, Segment) + + // PrecendingCharacter returns a character just before current internal pointer. + PrecendingCharacter() rune + + // Value returns a value of the given segment. + Value(Segment) []byte + + // LineOffset returns a distance from the line head to current position. + LineOffset() int + + // Position returns current line number and position. + Position() (int, Segment) + + // SetPosition sets current line number and position. + SetPosition(int, Segment) + + // SetPadding sets padding to the reader. + SetPadding(int) + + // Advance advances the internal pointer. + Advance(int) + + // AdvanceAndSetPadding advances the internal pointer and add padding to the + // reader. + AdvanceAndSetPadding(int, int) + + // AdvanceLine advances the internal pointer to the next line head. + AdvanceLine() + + // SkipSpaces skips space characters and returns a non-blank line. + // If it reaches EOF, returns false. + SkipSpaces() (Segment, int, bool) + + // SkipSpaces skips blank lines and returns a non-blank line. + // If it reaches EOF, returns false. + SkipBlankLines() (Segment, int, bool) + + // Match performs regular expression matching to current line. + Match(reg *regexp.Regexp) bool + + // Match performs regular expression searching to current line. + FindSubMatch(reg *regexp.Regexp) [][]byte + + // FindClosure finds corresponding closure. + FindClosure(opener, closer byte, options FindClosureOptions) (*Segments, bool) +} + +// FindClosureOptions is options for Reader.FindClosure +type FindClosureOptions struct { + // CodeSpan is a flag for the FindClosure. If this is set to true, + // FindClosure ignores closers in codespans. + CodeSpan bool + + // Nesting is a flag for the FindClosure. If this is set to true, + // FindClosure allows nesting. + Nesting bool + + // Newline is a flag for the FindClosure. If this is set to true, + // FindClosure searches for a closer over multiple lines. + Newline bool + + // Advance is a flag for the FindClosure. If this is set to true, + // FindClosure advances pointers when closer is found. + Advance bool +} + +type reader struct { + source []byte + sourceLength int + line int + peekedLine []byte + pos Segment + head int + lineOffset int +} + +// NewReader return a new Reader that can read UTF-8 bytes . +func NewReader(source []byte) Reader { + r := &reader{ + source: source, + sourceLength: len(source), + } + r.ResetPosition() + return r +} + +func (r *reader) FindClosure(opener, closer byte, options FindClosureOptions) (*Segments, bool) { + return findClosureReader(r, opener, closer, options) +} + +func (r *reader) ResetPosition() { + r.line = -1 + r.head = 0 + r.lineOffset = -1 + r.AdvanceLine() +} + +func (r *reader) Source() []byte { + return r.source +} + +func (r *reader) Value(seg Segment) []byte { + return seg.Value(r.source) +} + +func (r *reader) Peek() byte { + if r.pos.Start >= 0 && r.pos.Start < r.sourceLength { + if r.pos.Padding != 0 { + return space[0] + } + return r.source[r.pos.Start] + } + return EOF +} + +func (r *reader) PeekLine() ([]byte, Segment) { + if r.pos.Start >= 0 && r.pos.Start < r.sourceLength { + if r.peekedLine == nil { + r.peekedLine = r.pos.Value(r.Source()) + } + return r.peekedLine, r.pos + } + return nil, r.pos +} + +// io.RuneReader interface +func (r *reader) ReadRune() (rune, int, error) { + return readRuneReader(r) +} + +func (r *reader) LineOffset() int { + if r.lineOffset < 0 { + v := 0 + for i := r.head; i < r.pos.Start; i++ { + if r.source[i] == '\t' { + v += util.TabWidth(v) + } else { + v++ + } + } + r.lineOffset = v - r.pos.Padding + } + return r.lineOffset +} + +func (r *reader) PrecendingCharacter() rune { + if r.pos.Start <= 0 { + if r.pos.Padding != 0 { + return rune(' ') + } + return rune('\n') + } + i := r.pos.Start - 1 + for ; i >= 0; i-- { + if utf8.RuneStart(r.source[i]) { + break + } + } + rn, _ := utf8.DecodeRune(r.source[i:]) + return rn +} + +func (r *reader) Advance(n int) { + r.lineOffset = -1 + if n < len(r.peekedLine) && r.pos.Padding == 0 { + r.pos.Start += n + r.peekedLine = nil + return + } + r.peekedLine = nil + l := r.sourceLength + for ; n > 0 && r.pos.Start < l; n-- { + if r.pos.Padding != 0 { + r.pos.Padding-- + continue + } + if r.source[r.pos.Start] == '\n' { + r.AdvanceLine() + continue + } + r.pos.Start++ + } +} + +func (r *reader) AdvanceAndSetPadding(n, padding int) { + r.Advance(n) + if padding > r.pos.Padding { + r.SetPadding(padding) + } +} + +func (r *reader) AdvanceLine() { + r.lineOffset = -1 + r.peekedLine = nil + r.pos.Start = r.pos.Stop + r.head = r.pos.Start + if r.pos.Start < 0 { + return + } + r.pos.Stop = r.sourceLength + for i := r.pos.Start; i < r.sourceLength; i++ { + c := r.source[i] + if c == '\n' { + r.pos.Stop = i + 1 + break + } + } + r.line++ + r.pos.Padding = 0 +} + +func (r *reader) Position() (int, Segment) { + return r.line, r.pos +} + +func (r *reader) SetPosition(line int, pos Segment) { + r.lineOffset = -1 + r.line = line + r.pos = pos +} + +func (r *reader) SetPadding(v int) { + r.pos.Padding = v +} + +func (r *reader) SkipSpaces() (Segment, int, bool) { + return skipSpacesReader(r) +} + +func (r *reader) SkipBlankLines() (Segment, int, bool) { + return skipBlankLinesReader(r) +} + +func (r *reader) Match(reg *regexp.Regexp) bool { + return matchReader(r, reg) +} + +func (r *reader) FindSubMatch(reg *regexp.Regexp) [][]byte { + return findSubMatchReader(r, reg) +} + +// A BlockReader interface is a reader that is optimized for Blocks. +type BlockReader interface { + Reader + // Reset resets current state and sets new segments to the reader. + Reset(segment *Segments) +} + +type blockReader struct { + source []byte + segments *Segments + segmentsLength int + line int + pos Segment + head int + last int + lineOffset int +} + +// NewBlockReader returns a new BlockReader. +func NewBlockReader(source []byte, segments *Segments) BlockReader { + r := &blockReader{ + source: source, + } + if segments != nil { + r.Reset(segments) + } + return r +} + +func (r *blockReader) FindClosure(opener, closer byte, options FindClosureOptions) (*Segments, bool) { + return findClosureReader(r, opener, closer, options) +} + +func (r *blockReader) ResetPosition() { + r.line = -1 + r.head = 0 + r.last = 0 + r.lineOffset = -1 + r.pos.Start = -1 + r.pos.Stop = -1 + r.pos.Padding = 0 + if r.segmentsLength > 0 { + last := r.segments.At(r.segmentsLength - 1) + r.last = last.Stop + } + r.AdvanceLine() +} + +func (r *blockReader) Reset(segments *Segments) { + r.segments = segments + r.segmentsLength = segments.Len() + r.ResetPosition() +} + +func (r *blockReader) Source() []byte { + return r.source +} + +func (r *blockReader) Value(seg Segment) []byte { + line := r.segmentsLength - 1 + ret := make([]byte, 0, seg.Stop-seg.Start+1) + for ; line >= 0; line-- { + if seg.Start >= r.segments.At(line).Start { + break + } + } + i := seg.Start + for ; line < r.segmentsLength; line++ { + s := r.segments.At(line) + if i < 0 { + i = s.Start + } + ret = s.ConcatPadding(ret) + for ; i < seg.Stop && i < s.Stop; i++ { + ret = append(ret, r.source[i]) + } + i = -1 + if s.Stop > seg.Stop { + break + } + } + return ret +} + +// io.RuneReader interface +func (r *blockReader) ReadRune() (rune, int, error) { + return readRuneReader(r) +} + +func (r *blockReader) PrecendingCharacter() rune { + if r.pos.Padding != 0 { + return rune(' ') + } + if r.segments.Len() < 1 { + return rune('\n') + } + firstSegment := r.segments.At(0) + if r.line == 0 && r.pos.Start <= firstSegment.Start { + return rune('\n') + } + l := len(r.source) + i := r.pos.Start - 1 + for ; i < l && i >= 0; i-- { + if utf8.RuneStart(r.source[i]) { + break + } + } + if i < 0 || i >= l { + return rune('\n') + } + rn, _ := utf8.DecodeRune(r.source[i:]) + return rn +} + +func (r *blockReader) LineOffset() int { + if r.lineOffset < 0 { + v := 0 + for i := r.head; i < r.pos.Start; i++ { + if r.source[i] == '\t' { + v += util.TabWidth(v) + } else { + v++ + } + } + r.lineOffset = v - r.pos.Padding + } + return r.lineOffset +} + +func (r *blockReader) Peek() byte { + if r.line < r.segmentsLength && r.pos.Start >= 0 && r.pos.Start < r.last { + if r.pos.Padding != 0 { + return space[0] + } + return r.source[r.pos.Start] + } + return EOF +} + +func (r *blockReader) PeekLine() ([]byte, Segment) { + if r.line < r.segmentsLength && r.pos.Start >= 0 && r.pos.Start < r.last { + return r.pos.Value(r.source), r.pos + } + return nil, r.pos +} + +func (r *blockReader) Advance(n int) { + r.lineOffset = -1 + + if n < r.pos.Stop-r.pos.Start && r.pos.Padding == 0 { + r.pos.Start += n + return + } + + for ; n > 0; n-- { + if r.pos.Padding != 0 { + r.pos.Padding-- + continue + } + if r.pos.Start >= r.pos.Stop-1 && r.pos.Stop < r.last { + r.AdvanceLine() + continue + } + r.pos.Start++ + } +} + +func (r *blockReader) AdvanceAndSetPadding(n, padding int) { + r.Advance(n) + if padding > r.pos.Padding { + r.SetPadding(padding) + } +} + +func (r *blockReader) AdvanceLine() { + r.SetPosition(r.line+1, NewSegment(invalidValue, invalidValue)) + r.head = r.pos.Start +} + +func (r *blockReader) Position() (int, Segment) { + return r.line, r.pos +} + +func (r *blockReader) SetPosition(line int, pos Segment) { + r.lineOffset = -1 + r.line = line + if pos.Start == invalidValue { + if r.line < r.segmentsLength { + s := r.segments.At(line) + r.head = s.Start + r.pos = s + } + } else { + r.pos = pos + if r.line < r.segmentsLength { + s := r.segments.At(line) + r.head = s.Start + } + } +} + +func (r *blockReader) SetPadding(v int) { + r.lineOffset = -1 + r.pos.Padding = v +} + +func (r *blockReader) SkipSpaces() (Segment, int, bool) { + return skipSpacesReader(r) +} + +func (r *blockReader) SkipBlankLines() (Segment, int, bool) { + return skipBlankLinesReader(r) +} + +func (r *blockReader) Match(reg *regexp.Regexp) bool { + return matchReader(r, reg) +} + +func (r *blockReader) FindSubMatch(reg *regexp.Regexp) [][]byte { + return findSubMatchReader(r, reg) +} + +func skipBlankLinesReader(r Reader) (Segment, int, bool) { + lines := 0 + for { + line, seg := r.PeekLine() + if line == nil { + return seg, lines, false + } + if util.IsBlank(line) { + lines++ + r.AdvanceLine() + } else { + return seg, lines, true + } + } +} + +func skipSpacesReader(r Reader) (Segment, int, bool) { + chars := 0 + for { + line, segment := r.PeekLine() + if line == nil { + return segment, chars, false + } + for i, c := range line { + if util.IsSpace(c) { + chars++ + r.Advance(1) + continue + } + return segment.WithStart(segment.Start + i + 1), chars, true + } + } +} + +func matchReader(r Reader, reg *regexp.Regexp) bool { + oldline, oldseg := r.Position() + match := reg.FindReaderSubmatchIndex(r) + r.SetPosition(oldline, oldseg) + if match == nil { + return false + } + r.Advance(match[1] - match[0]) + return true +} + +func findSubMatchReader(r Reader, reg *regexp.Regexp) [][]byte { + oldline, oldseg := r.Position() + match := reg.FindReaderSubmatchIndex(r) + r.SetPosition(oldline, oldseg) + if match == nil { + return nil + } + runes := make([]rune, 0, match[1]-match[0]) + for i := 0; i < match[1]; { + r, size, _ := readRuneReader(r) + i += size + runes = append(runes, r) + } + result := [][]byte{} + for i := 0; i < len(match); i += 2 { + result = append(result, []byte(string(runes[match[i]:match[i+1]]))) + } + + r.SetPosition(oldline, oldseg) + r.Advance(match[1] - match[0]) + return result +} + +func readRuneReader(r Reader) (rune, int, error) { + line, _ := r.PeekLine() + if line == nil { + return 0, 0, io.EOF + } + rn, size := utf8.DecodeRune(line) + if rn == utf8.RuneError { + return 0, 0, io.EOF + } + r.Advance(size) + return rn, size, nil +} + +func findClosureReader(r Reader, opener, closer byte, opts FindClosureOptions) (*Segments, bool) { + opened := 1 + codeSpanOpener := 0 + closed := false + orgline, orgpos := r.Position() + var ret *Segments + + for { + bs, seg := r.PeekLine() + if bs == nil { + goto end + } + i := 0 + for i < len(bs) { + c := bs[i] + if opts.CodeSpan && codeSpanOpener != 0 && c == '`' { + codeSpanCloser := 0 + for ; i < len(bs); i++ { + if bs[i] == '`' { + codeSpanCloser++ + } else { + i-- + break + } + } + if codeSpanCloser == codeSpanOpener { + codeSpanOpener = 0 + } + } else if codeSpanOpener == 0 && c == '\\' && i < len(bs)-1 && util.IsPunct(bs[i+1]) { + i += 2 + continue + } else if opts.CodeSpan && codeSpanOpener == 0 && c == '`' { + for ; i < len(bs); i++ { + if bs[i] == '`' { + codeSpanOpener++ + } else { + i-- + break + } + } + } else if (opts.CodeSpan && codeSpanOpener == 0) || !opts.CodeSpan { + if c == closer { + opened-- + if opened == 0 { + if ret == nil { + ret = NewSegments() + } + ret.Append(seg.WithStop(seg.Start + i)) + r.Advance(i + 1) + closed = true + goto end + } + } else if c == opener { + if !opts.Nesting { + goto end + } + opened++ + } + } + i++ + } + if !opts.Newline { + goto end + } + r.AdvanceLine() + if ret == nil { + ret = NewSegments() + } + ret.Append(seg) + } +end: + if !opts.Advance { + r.SetPosition(orgline, orgpos) + } + if closed { + return ret, true + } + return nil, false +} diff --git a/vendor/github.com/yuin/goldmark/text/segment.go b/vendor/github.com/yuin/goldmark/text/segment.go new file mode 100644 index 00000000..badd4bc8 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/text/segment.go @@ -0,0 +1,209 @@ +package text + +import ( + "bytes" + "github.com/yuin/goldmark/util" +) + +var space = []byte(" ") + +// A Segment struct holds information about source positions. +type Segment struct { + // Start is a start position of the segment. + Start int + + // Stop is a stop position of the segment. + // This value should be excluded. + Stop int + + // Padding is a padding length of the segment. + Padding int +} + +// NewSegment return a new Segment. +func NewSegment(start, stop int) Segment { + return Segment{ + Start: start, + Stop: stop, + Padding: 0, + } +} + +// NewSegmentPadding returns a new Segment with the given padding. +func NewSegmentPadding(start, stop, n int) Segment { + return Segment{ + Start: start, + Stop: stop, + Padding: n, + } +} + +// Value returns a value of the segment. +func (t *Segment) Value(buffer []byte) []byte { + if t.Padding == 0 { + return buffer[t.Start:t.Stop] + } + result := make([]byte, 0, t.Padding+t.Stop-t.Start+1) + result = append(result, bytes.Repeat(space, t.Padding)...) + return append(result, buffer[t.Start:t.Stop]...) +} + +// Len returns a length of the segment. +func (t *Segment) Len() int { + return t.Stop - t.Start + t.Padding +} + +// Between returns a segment between this segment and the given segment. +func (t *Segment) Between(other Segment) Segment { + if t.Stop != other.Stop { + panic("invalid state") + } + return NewSegmentPadding( + t.Start, + other.Start, + t.Padding-other.Padding, + ) +} + +// IsEmpty returns true if this segment is empty, otherwise false. +func (t *Segment) IsEmpty() bool { + return t.Start >= t.Stop && t.Padding == 0 +} + +// TrimRightSpace returns a new segment by slicing off all trailing +// space characters. +func (t *Segment) TrimRightSpace(buffer []byte) Segment { + v := buffer[t.Start:t.Stop] + l := util.TrimRightSpaceLength(v) + if l == len(v) { + return NewSegment(t.Start, t.Start) + } + return NewSegmentPadding(t.Start, t.Stop-l, t.Padding) +} + +// TrimLeftSpace returns a new segment by slicing off all leading +// space characters including padding. +func (t *Segment) TrimLeftSpace(buffer []byte) Segment { + v := buffer[t.Start:t.Stop] + l := util.TrimLeftSpaceLength(v) + return NewSegment(t.Start+l, t.Stop) +} + +// TrimLeftSpaceWidth returns a new segment by slicing off leading space +// characters until the given width. +func (t *Segment) TrimLeftSpaceWidth(width int, buffer []byte) Segment { + padding := t.Padding + for ; width > 0; width-- { + if padding == 0 { + break + } + padding-- + } + if width == 0 { + return NewSegmentPadding(t.Start, t.Stop, padding) + } + text := buffer[t.Start:t.Stop] + start := t.Start + for _, c := range text { + if start >= t.Stop-1 || width <= 0 { + break + } + if c == ' ' { + width-- + } else if c == '\t' { + width -= 4 + } else { + break + } + start++ + } + if width < 0 { + padding = width * -1 + } + return NewSegmentPadding(start, t.Stop, padding) +} + +// WithStart returns a new Segment with same value except Start. +func (t *Segment) WithStart(v int) Segment { + return NewSegmentPadding(v, t.Stop, t.Padding) +} + +// WithStop returns a new Segment with same value except Stop. +func (t *Segment) WithStop(v int) Segment { + return NewSegmentPadding(t.Start, v, t.Padding) +} + +// ConcatPadding concats the padding to the given slice. +func (t *Segment) ConcatPadding(v []byte) []byte { + if t.Padding > 0 { + return append(v, bytes.Repeat(space, t.Padding)...) + } + return v +} + +// Segments is a collection of the Segment. +type Segments struct { + values []Segment +} + +// NewSegments return a new Segments. +func NewSegments() *Segments { + return &Segments{ + values: nil, + } +} + +// Append appends the given segment after the tail of the collection. +func (s *Segments) Append(t Segment) { + if s.values == nil { + s.values = make([]Segment, 0, 20) + } + s.values = append(s.values, t) +} + +// AppendAll appends all elements of given segments after the tail of the collection. +func (s *Segments) AppendAll(t []Segment) { + if s.values == nil { + s.values = make([]Segment, 0, 20) + } + s.values = append(s.values, t...) +} + +// Len returns the length of the collection. +func (s *Segments) Len() int { + if s.values == nil { + return 0 + } + return len(s.values) +} + +// At returns a segment at the given index. +func (s *Segments) At(i int) Segment { + return s.values[i] +} + +// Set sets the given Segment. +func (s *Segments) Set(i int, v Segment) { + s.values[i] = v +} + +// SetSliced replace the collection with a subsliced value. +func (s *Segments) SetSliced(lo, hi int) { + s.values = s.values[lo:hi] +} + +// Sliced returns a subslice of the collection. +func (s *Segments) Sliced(lo, hi int) []Segment { + return s.values[lo:hi] +} + +// Clear delete all element of the collection. +func (s *Segments) Clear() { + s.values = nil +} + +// Unshift insert the given Segment to head of the collection. +func (s *Segments) Unshift(v Segment) { + s.values = append(s.values[0:1], s.values[0:]...) + s.values[0] = v +} diff --git a/vendor/github.com/yuin/goldmark/util/html5entities.go b/vendor/github.com/yuin/goldmark/util/html5entities.go new file mode 100644 index 00000000..b8e00a91 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/util/html5entities.go @@ -0,0 +1,2142 @@ +package util + +// An HTML5Entity struct represents HTML5 entitites. +type HTML5Entity struct { + Name string + CodePoints []int + Characters []byte +} + +// LookUpHTML5EntityByName returns (an HTML5Entity, true) if an entity named +// given name is found, otherwise (nil, false) +func LookUpHTML5EntityByName(name string) (*HTML5Entity, bool) { + v, ok := html5entities[name] + return v, ok +} + +var html5entities = map[string]*HTML5Entity{ + "AElig": {Name: "AElig", CodePoints: []int{198}, Characters: []byte{0xc3, 0x86}}, + "AMP": {Name: "AMP", CodePoints: []int{38}, Characters: []byte{0x26}}, + "Aacute": {Name: "Aacute", CodePoints: []int{193}, Characters: []byte{0xc3, 0x81}}, + "Acirc": {Name: "Acirc", CodePoints: []int{194}, Characters: []byte{0xc3, 0x82}}, + "Acy": {Name: "Acy", CodePoints: []int{1040}, Characters: []byte{0xd0, 0x90}}, + "Afr": {Name: "Afr", CodePoints: []int{120068}, Characters: []byte{0xf0, 0x9d, 0x94, 0x84}}, + "Agrave": {Name: "Agrave", CodePoints: []int{192}, Characters: []byte{0xc3, 0x80}}, + "Alpha": {Name: "Alpha", CodePoints: []int{913}, Characters: []byte{0xce, 0x91}}, + "Amacr": {Name: "Amacr", CodePoints: []int{256}, Characters: []byte{0xc4, 0x80}}, + "And": {Name: "And", CodePoints: []int{10835}, Characters: []byte{0xe2, 0xa9, 0x93}}, + "Aogon": {Name: "Aogon", CodePoints: []int{260}, Characters: []byte{0xc4, 0x84}}, + "Aopf": {Name: "Aopf", CodePoints: []int{120120}, Characters: []byte{0xf0, 0x9d, 0x94, 0xb8}}, + "ApplyFunction": {Name: "ApplyFunction", CodePoints: []int{8289}, Characters: []byte{0xe2, 0x81, 0xa1}}, + "Aring": {Name: "Aring", CodePoints: []int{197}, Characters: []byte{0xc3, 0x85}}, + "Ascr": {Name: "Ascr", CodePoints: []int{119964}, Characters: []byte{0xf0, 0x9d, 0x92, 0x9c}}, + "Assign": {Name: "Assign", CodePoints: []int{8788}, Characters: []byte{0xe2, 0x89, 0x94}}, + "Atilde": {Name: "Atilde", CodePoints: []int{195}, Characters: []byte{0xc3, 0x83}}, + "Auml": {Name: "Auml", CodePoints: []int{196}, Characters: []byte{0xc3, 0x84}}, + "Backslash": {Name: "Backslash", CodePoints: []int{8726}, Characters: []byte{0xe2, 0x88, 0x96}}, + "Barv": {Name: "Barv", CodePoints: []int{10983}, Characters: []byte{0xe2, 0xab, 0xa7}}, + "Barwed": {Name: "Barwed", CodePoints: []int{8966}, Characters: []byte{0xe2, 0x8c, 0x86}}, + "Bcy": {Name: "Bcy", CodePoints: []int{1041}, Characters: []byte{0xd0, 0x91}}, + "Because": {Name: "Because", CodePoints: []int{8757}, Characters: []byte{0xe2, 0x88, 0xb5}}, + "Bernoullis": {Name: "Bernoullis", CodePoints: []int{8492}, Characters: []byte{0xe2, 0x84, 0xac}}, + "Beta": {Name: "Beta", CodePoints: []int{914}, Characters: []byte{0xce, 0x92}}, + "Bfr": {Name: "Bfr", CodePoints: []int{120069}, Characters: []byte{0xf0, 0x9d, 0x94, 0x85}}, + "Bopf": {Name: "Bopf", CodePoints: []int{120121}, Characters: []byte{0xf0, 0x9d, 0x94, 0xb9}}, + "Breve": {Name: "Breve", CodePoints: []int{728}, Characters: []byte{0xcb, 0x98}}, + "Bscr": {Name: "Bscr", CodePoints: []int{8492}, Characters: []byte{0xe2, 0x84, 0xac}}, + "Bumpeq": {Name: "Bumpeq", CodePoints: []int{8782}, Characters: []byte{0xe2, 0x89, 0x8e}}, + "CHcy": {Name: "CHcy", CodePoints: []int{1063}, Characters: []byte{0xd0, 0xa7}}, + "COPY": {Name: "COPY", CodePoints: []int{169}, Characters: []byte{0xc2, 0xa9}}, + "Cacute": {Name: "Cacute", CodePoints: []int{262}, Characters: []byte{0xc4, 0x86}}, + "Cap": {Name: "Cap", CodePoints: []int{8914}, Characters: []byte{0xe2, 0x8b, 0x92}}, + "CapitalDifferentialD": {Name: "CapitalDifferentialD", CodePoints: []int{8517}, Characters: []byte{0xe2, 0x85, 0x85}}, + "Cayleys": {Name: "Cayleys", CodePoints: []int{8493}, Characters: []byte{0xe2, 0x84, 0xad}}, + "Ccaron": {Name: "Ccaron", CodePoints: []int{268}, Characters: []byte{0xc4, 0x8c}}, + "Ccedil": {Name: "Ccedil", CodePoints: []int{199}, Characters: []byte{0xc3, 0x87}}, + "Ccirc": {Name: "Ccirc", CodePoints: []int{264}, Characters: []byte{0xc4, 0x88}}, + "Cconint": {Name: "Cconint", CodePoints: []int{8752}, Characters: []byte{0xe2, 0x88, 0xb0}}, + "Cdot": {Name: "Cdot", CodePoints: []int{266}, Characters: []byte{0xc4, 0x8a}}, + "Cedilla": {Name: "Cedilla", CodePoints: []int{184}, Characters: []byte{0xc2, 0xb8}}, + "CenterDot": {Name: "CenterDot", CodePoints: []int{183}, Characters: []byte{0xc2, 0xb7}}, + "Cfr": {Name: "Cfr", CodePoints: []int{8493}, Characters: []byte{0xe2, 0x84, 0xad}}, + "Chi": {Name: "Chi", CodePoints: []int{935}, Characters: []byte{0xce, 0xa7}}, + "CircleDot": {Name: "CircleDot", CodePoints: []int{8857}, Characters: []byte{0xe2, 0x8a, 0x99}}, + "CircleMinus": {Name: "CircleMinus", CodePoints: []int{8854}, Characters: []byte{0xe2, 0x8a, 0x96}}, + "CirclePlus": {Name: "CirclePlus", CodePoints: []int{8853}, Characters: []byte{0xe2, 0x8a, 0x95}}, + "CircleTimes": {Name: "CircleTimes", CodePoints: []int{8855}, Characters: []byte{0xe2, 0x8a, 0x97}}, + "ClockwiseContourIntegral": {Name: "ClockwiseContourIntegral", CodePoints: []int{8754}, Characters: []byte{0xe2, 0x88, 0xb2}}, + "CloseCurlyDoubleQuote": {Name: "CloseCurlyDoubleQuote", CodePoints: []int{8221}, Characters: []byte{0xe2, 0x80, 0x9d}}, + "CloseCurlyQuote": {Name: "CloseCurlyQuote", CodePoints: []int{8217}, Characters: []byte{0xe2, 0x80, 0x99}}, + "Colon": {Name: "Colon", CodePoints: []int{8759}, Characters: []byte{0xe2, 0x88, 0xb7}}, + "Colone": {Name: "Colone", CodePoints: []int{10868}, Characters: []byte{0xe2, 0xa9, 0xb4}}, + "Congruent": {Name: "Congruent", CodePoints: []int{8801}, Characters: []byte{0xe2, 0x89, 0xa1}}, + "Conint": {Name: "Conint", CodePoints: []int{8751}, Characters: []byte{0xe2, 0x88, 0xaf}}, + "ContourIntegral": {Name: "ContourIntegral", CodePoints: []int{8750}, Characters: []byte{0xe2, 0x88, 0xae}}, + "Copf": {Name: "Copf", CodePoints: []int{8450}, Characters: []byte{0xe2, 0x84, 0x82}}, + "Coproduct": {Name: "Coproduct", CodePoints: []int{8720}, Characters: []byte{0xe2, 0x88, 0x90}}, + "CounterClockwiseContourIntegral": {Name: "CounterClockwiseContourIntegral", CodePoints: []int{8755}, Characters: []byte{0xe2, 0x88, 0xb3}}, + "Cross": {Name: "Cross", CodePoints: []int{10799}, Characters: []byte{0xe2, 0xa8, 0xaf}}, + "Cscr": {Name: "Cscr", CodePoints: []int{119966}, Characters: []byte{0xf0, 0x9d, 0x92, 0x9e}}, + "Cup": {Name: "Cup", CodePoints: []int{8915}, Characters: []byte{0xe2, 0x8b, 0x93}}, + "CupCap": {Name: "CupCap", CodePoints: []int{8781}, Characters: []byte{0xe2, 0x89, 0x8d}}, + "DD": {Name: "DD", CodePoints: []int{8517}, Characters: []byte{0xe2, 0x85, 0x85}}, + "DDotrahd": {Name: "DDotrahd", CodePoints: []int{10513}, Characters: []byte{0xe2, 0xa4, 0x91}}, + "DJcy": {Name: "DJcy", CodePoints: []int{1026}, Characters: []byte{0xd0, 0x82}}, + "DScy": {Name: "DScy", CodePoints: []int{1029}, Characters: []byte{0xd0, 0x85}}, + "DZcy": {Name: "DZcy", CodePoints: []int{1039}, Characters: []byte{0xd0, 0x8f}}, + "Dagger": {Name: "Dagger", CodePoints: []int{8225}, Characters: []byte{0xe2, 0x80, 0xa1}}, + "Darr": {Name: "Darr", CodePoints: []int{8609}, Characters: []byte{0xe2, 0x86, 0xa1}}, + "Dashv": {Name: "Dashv", CodePoints: []int{10980}, Characters: []byte{0xe2, 0xab, 0xa4}}, + "Dcaron": {Name: "Dcaron", CodePoints: []int{270}, Characters: []byte{0xc4, 0x8e}}, + "Dcy": {Name: "Dcy", CodePoints: []int{1044}, Characters: []byte{0xd0, 0x94}}, + "Del": {Name: "Del", CodePoints: []int{8711}, Characters: []byte{0xe2, 0x88, 0x87}}, + "Delta": {Name: "Delta", CodePoints: []int{916}, Characters: []byte{0xce, 0x94}}, + "Dfr": {Name: "Dfr", CodePoints: []int{120071}, Characters: []byte{0xf0, 0x9d, 0x94, 0x87}}, + "DiacriticalAcute": {Name: "DiacriticalAcute", CodePoints: []int{180}, Characters: []byte{0xc2, 0xb4}}, + "DiacriticalDot": {Name: "DiacriticalDot", CodePoints: []int{729}, Characters: []byte{0xcb, 0x99}}, + "DiacriticalDoubleAcute": {Name: "DiacriticalDoubleAcute", CodePoints: []int{733}, Characters: []byte{0xcb, 0x9d}}, + "DiacriticalGrave": {Name: "DiacriticalGrave", CodePoints: []int{96}, Characters: []byte{0x60}}, + "DiacriticalTilde": {Name: "DiacriticalTilde", CodePoints: []int{732}, Characters: []byte{0xcb, 0x9c}}, + "Diamond": {Name: "Diamond", CodePoints: []int{8900}, Characters: []byte{0xe2, 0x8b, 0x84}}, + "DifferentialD": {Name: "DifferentialD", CodePoints: []int{8518}, Characters: []byte{0xe2, 0x85, 0x86}}, + "Dopf": {Name: "Dopf", CodePoints: []int{120123}, Characters: []byte{0xf0, 0x9d, 0x94, 0xbb}}, + "Dot": {Name: "Dot", CodePoints: []int{168}, Characters: []byte{0xc2, 0xa8}}, + "DotDot": {Name: "DotDot", CodePoints: []int{8412}, Characters: []byte{0xe2, 0x83, 0x9c}}, + "DotEqual": {Name: "DotEqual", CodePoints: []int{8784}, Characters: []byte{0xe2, 0x89, 0x90}}, + "DoubleContourIntegral": {Name: "DoubleContourIntegral", CodePoints: []int{8751}, Characters: []byte{0xe2, 0x88, 0xaf}}, + "DoubleDot": {Name: "DoubleDot", CodePoints: []int{168}, Characters: []byte{0xc2, 0xa8}}, + "DoubleDownArrow": {Name: "DoubleDownArrow", CodePoints: []int{8659}, Characters: []byte{0xe2, 0x87, 0x93}}, + "DoubleLeftArrow": {Name: "DoubleLeftArrow", CodePoints: []int{8656}, Characters: []byte{0xe2, 0x87, 0x90}}, + "DoubleLeftRightArrow": {Name: "DoubleLeftRightArrow", CodePoints: []int{8660}, Characters: []byte{0xe2, 0x87, 0x94}}, + "DoubleLeftTee": {Name: "DoubleLeftTee", CodePoints: []int{10980}, Characters: []byte{0xe2, 0xab, 0xa4}}, + "DoubleLongLeftArrow": {Name: "DoubleLongLeftArrow", CodePoints: []int{10232}, Characters: []byte{0xe2, 0x9f, 0xb8}}, + "DoubleLongLeftRightArrow": {Name: "DoubleLongLeftRightArrow", CodePoints: []int{10234}, Characters: []byte{0xe2, 0x9f, 0xba}}, + "DoubleLongRightArrow": {Name: "DoubleLongRightArrow", CodePoints: []int{10233}, Characters: []byte{0xe2, 0x9f, 0xb9}}, + "DoubleRightArrow": {Name: "DoubleRightArrow", CodePoints: []int{8658}, Characters: []byte{0xe2, 0x87, 0x92}}, + "DoubleRightTee": {Name: "DoubleRightTee", CodePoints: []int{8872}, Characters: []byte{0xe2, 0x8a, 0xa8}}, + "DoubleUpArrow": {Name: "DoubleUpArrow", CodePoints: []int{8657}, Characters: []byte{0xe2, 0x87, 0x91}}, + "DoubleUpDownArrow": {Name: "DoubleUpDownArrow", CodePoints: []int{8661}, Characters: []byte{0xe2, 0x87, 0x95}}, + "DoubleVerticalBar": {Name: "DoubleVerticalBar", CodePoints: []int{8741}, Characters: []byte{0xe2, 0x88, 0xa5}}, + "DownArrow": {Name: "DownArrow", CodePoints: []int{8595}, Characters: []byte{0xe2, 0x86, 0x93}}, + "DownArrowBar": {Name: "DownArrowBar", CodePoints: []int{10515}, Characters: []byte{0xe2, 0xa4, 0x93}}, + "DownArrowUpArrow": {Name: "DownArrowUpArrow", CodePoints: []int{8693}, Characters: []byte{0xe2, 0x87, 0xb5}}, + "DownBreve": {Name: "DownBreve", CodePoints: []int{785}, Characters: []byte{0xcc, 0x91}}, + "DownLeftRightVector": {Name: "DownLeftRightVector", CodePoints: []int{10576}, Characters: []byte{0xe2, 0xa5, 0x90}}, + "DownLeftTeeVector": {Name: "DownLeftTeeVector", CodePoints: []int{10590}, Characters: []byte{0xe2, 0xa5, 0x9e}}, + "DownLeftVector": {Name: "DownLeftVector", CodePoints: []int{8637}, Characters: []byte{0xe2, 0x86, 0xbd}}, + "DownLeftVectorBar": {Name: "DownLeftVectorBar", CodePoints: []int{10582}, Characters: []byte{0xe2, 0xa5, 0x96}}, + "DownRightTeeVector": {Name: "DownRightTeeVector", CodePoints: []int{10591}, Characters: []byte{0xe2, 0xa5, 0x9f}}, + "DownRightVector": {Name: "DownRightVector", CodePoints: []int{8641}, Characters: []byte{0xe2, 0x87, 0x81}}, + "DownRightVectorBar": {Name: "DownRightVectorBar", CodePoints: []int{10583}, Characters: []byte{0xe2, 0xa5, 0x97}}, + "DownTee": {Name: "DownTee", CodePoints: []int{8868}, Characters: []byte{0xe2, 0x8a, 0xa4}}, + "DownTeeArrow": {Name: "DownTeeArrow", CodePoints: []int{8615}, Characters: []byte{0xe2, 0x86, 0xa7}}, + "Downarrow": {Name: "Downarrow", CodePoints: []int{8659}, Characters: []byte{0xe2, 0x87, 0x93}}, + "Dscr": {Name: "Dscr", CodePoints: []int{119967}, Characters: []byte{0xf0, 0x9d, 0x92, 0x9f}}, + "Dstrok": {Name: "Dstrok", CodePoints: []int{272}, Characters: []byte{0xc4, 0x90}}, + "ENG": {Name: "ENG", CodePoints: []int{330}, Characters: []byte{0xc5, 0x8a}}, + "ETH": {Name: "ETH", CodePoints: []int{208}, Characters: []byte{0xc3, 0x90}}, + "Eacute": {Name: "Eacute", CodePoints: []int{201}, Characters: []byte{0xc3, 0x89}}, + "Ecaron": {Name: "Ecaron", CodePoints: []int{282}, Characters: []byte{0xc4, 0x9a}}, + "Ecirc": {Name: "Ecirc", CodePoints: []int{202}, Characters: []byte{0xc3, 0x8a}}, + "Ecy": {Name: "Ecy", CodePoints: []int{1069}, Characters: []byte{0xd0, 0xad}}, + "Edot": {Name: "Edot", CodePoints: []int{278}, Characters: []byte{0xc4, 0x96}}, + "Efr": {Name: "Efr", CodePoints: []int{120072}, Characters: []byte{0xf0, 0x9d, 0x94, 0x88}}, + "Egrave": {Name: "Egrave", CodePoints: []int{200}, Characters: []byte{0xc3, 0x88}}, + "Element": {Name: "Element", CodePoints: []int{8712}, Characters: []byte{0xe2, 0x88, 0x88}}, + "Emacr": {Name: "Emacr", CodePoints: []int{274}, Characters: []byte{0xc4, 0x92}}, + "EmptySmallSquare": {Name: "EmptySmallSquare", CodePoints: []int{9723}, Characters: []byte{0xe2, 0x97, 0xbb}}, + "EmptyVerySmallSquare": {Name: "EmptyVerySmallSquare", CodePoints: []int{9643}, Characters: []byte{0xe2, 0x96, 0xab}}, + "Eogon": {Name: "Eogon", CodePoints: []int{280}, Characters: []byte{0xc4, 0x98}}, + "Eopf": {Name: "Eopf", CodePoints: []int{120124}, Characters: []byte{0xf0, 0x9d, 0x94, 0xbc}}, + "Epsilon": {Name: "Epsilon", CodePoints: []int{917}, Characters: []byte{0xce, 0x95}}, + "Equal": {Name: "Equal", CodePoints: []int{10869}, Characters: []byte{0xe2, 0xa9, 0xb5}}, + "EqualTilde": {Name: "EqualTilde", CodePoints: []int{8770}, Characters: []byte{0xe2, 0x89, 0x82}}, + "Equilibrium": {Name: "Equilibrium", CodePoints: []int{8652}, Characters: []byte{0xe2, 0x87, 0x8c}}, + "Escr": {Name: "Escr", CodePoints: []int{8496}, Characters: []byte{0xe2, 0x84, 0xb0}}, + "Esim": {Name: "Esim", CodePoints: []int{10867}, Characters: []byte{0xe2, 0xa9, 0xb3}}, + "Eta": {Name: "Eta", CodePoints: []int{919}, Characters: []byte{0xce, 0x97}}, + "Euml": {Name: "Euml", CodePoints: []int{203}, Characters: []byte{0xc3, 0x8b}}, + "Exists": {Name: "Exists", CodePoints: []int{8707}, Characters: []byte{0xe2, 0x88, 0x83}}, + "ExponentialE": {Name: "ExponentialE", CodePoints: []int{8519}, Characters: []byte{0xe2, 0x85, 0x87}}, + "Fcy": {Name: "Fcy", CodePoints: []int{1060}, Characters: []byte{0xd0, 0xa4}}, + "Ffr": {Name: "Ffr", CodePoints: []int{120073}, Characters: []byte{0xf0, 0x9d, 0x94, 0x89}}, + "FilledSmallSquare": {Name: "FilledSmallSquare", CodePoints: []int{9724}, Characters: []byte{0xe2, 0x97, 0xbc}}, + "FilledVerySmallSquare": {Name: "FilledVerySmallSquare", CodePoints: []int{9642}, Characters: []byte{0xe2, 0x96, 0xaa}}, + "Fopf": {Name: "Fopf", CodePoints: []int{120125}, Characters: []byte{0xf0, 0x9d, 0x94, 0xbd}}, + "ForAll": {Name: "ForAll", CodePoints: []int{8704}, Characters: []byte{0xe2, 0x88, 0x80}}, + "Fouriertrf": {Name: "Fouriertrf", CodePoints: []int{8497}, Characters: []byte{0xe2, 0x84, 0xb1}}, + "Fscr": {Name: "Fscr", CodePoints: []int{8497}, Characters: []byte{0xe2, 0x84, 0xb1}}, + "GJcy": {Name: "GJcy", CodePoints: []int{1027}, Characters: []byte{0xd0, 0x83}}, + "GT": {Name: "GT", CodePoints: []int{62}, Characters: []byte{0x3e}}, + "Gamma": {Name: "Gamma", CodePoints: []int{915}, Characters: []byte{0xce, 0x93}}, + "Gammad": {Name: "Gammad", CodePoints: []int{988}, Characters: []byte{0xcf, 0x9c}}, + "Gbreve": {Name: "Gbreve", CodePoints: []int{286}, Characters: []byte{0xc4, 0x9e}}, + "Gcedil": {Name: "Gcedil", CodePoints: []int{290}, Characters: []byte{0xc4, 0xa2}}, + "Gcirc": {Name: "Gcirc", CodePoints: []int{284}, Characters: []byte{0xc4, 0x9c}}, + "Gcy": {Name: "Gcy", CodePoints: []int{1043}, Characters: []byte{0xd0, 0x93}}, + "Gdot": {Name: "Gdot", CodePoints: []int{288}, Characters: []byte{0xc4, 0xa0}}, + "Gfr": {Name: "Gfr", CodePoints: []int{120074}, Characters: []byte{0xf0, 0x9d, 0x94, 0x8a}}, + "Gg": {Name: "Gg", CodePoints: []int{8921}, Characters: []byte{0xe2, 0x8b, 0x99}}, + "Gopf": {Name: "Gopf", CodePoints: []int{120126}, Characters: []byte{0xf0, 0x9d, 0x94, 0xbe}}, + "GreaterEqual": {Name: "GreaterEqual", CodePoints: []int{8805}, Characters: []byte{0xe2, 0x89, 0xa5}}, + "GreaterEqualLess": {Name: "GreaterEqualLess", CodePoints: []int{8923}, Characters: []byte{0xe2, 0x8b, 0x9b}}, + "GreaterFullEqual": {Name: "GreaterFullEqual", CodePoints: []int{8807}, Characters: []byte{0xe2, 0x89, 0xa7}}, + "GreaterGreater": {Name: "GreaterGreater", CodePoints: []int{10914}, Characters: []byte{0xe2, 0xaa, 0xa2}}, + "GreaterLess": {Name: "GreaterLess", CodePoints: []int{8823}, Characters: []byte{0xe2, 0x89, 0xb7}}, + "GreaterSlantEqual": {Name: "GreaterSlantEqual", CodePoints: []int{10878}, Characters: []byte{0xe2, 0xa9, 0xbe}}, + "GreaterTilde": {Name: "GreaterTilde", CodePoints: []int{8819}, Characters: []byte{0xe2, 0x89, 0xb3}}, + "Gscr": {Name: "Gscr", CodePoints: []int{119970}, Characters: []byte{0xf0, 0x9d, 0x92, 0xa2}}, + "Gt": {Name: "Gt", CodePoints: []int{8811}, Characters: []byte{0xe2, 0x89, 0xab}}, + "HARDcy": {Name: "HARDcy", CodePoints: []int{1066}, Characters: []byte{0xd0, 0xaa}}, + "Hacek": {Name: "Hacek", CodePoints: []int{711}, Characters: []byte{0xcb, 0x87}}, + "Hat": {Name: "Hat", CodePoints: []int{94}, Characters: []byte{0x5e}}, + "Hcirc": {Name: "Hcirc", CodePoints: []int{292}, Characters: []byte{0xc4, 0xa4}}, + "Hfr": {Name: "Hfr", CodePoints: []int{8460}, Characters: []byte{0xe2, 0x84, 0x8c}}, + "HilbertSpace": {Name: "HilbertSpace", CodePoints: []int{8459}, Characters: []byte{0xe2, 0x84, 0x8b}}, + "Hopf": {Name: "Hopf", CodePoints: []int{8461}, Characters: []byte{0xe2, 0x84, 0x8d}}, + "HorizontalLine": {Name: "HorizontalLine", CodePoints: []int{9472}, Characters: []byte{0xe2, 0x94, 0x80}}, + "Hscr": {Name: "Hscr", CodePoints: []int{8459}, Characters: []byte{0xe2, 0x84, 0x8b}}, + "Hstrok": {Name: "Hstrok", CodePoints: []int{294}, Characters: []byte{0xc4, 0xa6}}, + "HumpDownHump": {Name: "HumpDownHump", CodePoints: []int{8782}, Characters: []byte{0xe2, 0x89, 0x8e}}, + "HumpEqual": {Name: "HumpEqual", CodePoints: []int{8783}, Characters: []byte{0xe2, 0x89, 0x8f}}, + "IEcy": {Name: "IEcy", CodePoints: []int{1045}, Characters: []byte{0xd0, 0x95}}, + "IJlig": {Name: "IJlig", CodePoints: []int{306}, Characters: []byte{0xc4, 0xb2}}, + "IOcy": {Name: "IOcy", CodePoints: []int{1025}, Characters: []byte{0xd0, 0x81}}, + "Iacute": {Name: "Iacute", CodePoints: []int{205}, Characters: []byte{0xc3, 0x8d}}, + "Icirc": {Name: "Icirc", CodePoints: []int{206}, Characters: []byte{0xc3, 0x8e}}, + "Icy": {Name: "Icy", CodePoints: []int{1048}, Characters: []byte{0xd0, 0x98}}, + "Idot": {Name: "Idot", CodePoints: []int{304}, Characters: []byte{0xc4, 0xb0}}, + "Ifr": {Name: "Ifr", CodePoints: []int{8465}, Characters: []byte{0xe2, 0x84, 0x91}}, + "Igrave": {Name: "Igrave", CodePoints: []int{204}, Characters: []byte{0xc3, 0x8c}}, + "Im": {Name: "Im", CodePoints: []int{8465}, Characters: []byte{0xe2, 0x84, 0x91}}, + "Imacr": {Name: "Imacr", CodePoints: []int{298}, Characters: []byte{0xc4, 0xaa}}, + "ImaginaryI": {Name: "ImaginaryI", CodePoints: []int{8520}, Characters: []byte{0xe2, 0x85, 0x88}}, + "Implies": {Name: "Implies", CodePoints: []int{8658}, Characters: []byte{0xe2, 0x87, 0x92}}, + "Int": {Name: "Int", CodePoints: []int{8748}, Characters: []byte{0xe2, 0x88, 0xac}}, + "Integral": {Name: "Integral", CodePoints: []int{8747}, Characters: []byte{0xe2, 0x88, 0xab}}, + "Intersection": {Name: "Intersection", CodePoints: []int{8898}, Characters: []byte{0xe2, 0x8b, 0x82}}, + "InvisibleComma": {Name: "InvisibleComma", CodePoints: []int{8291}, Characters: []byte{0xe2, 0x81, 0xa3}}, + "InvisibleTimes": {Name: "InvisibleTimes", CodePoints: []int{8290}, Characters: []byte{0xe2, 0x81, 0xa2}}, + "Iogon": {Name: "Iogon", CodePoints: []int{302}, Characters: []byte{0xc4, 0xae}}, + "Iopf": {Name: "Iopf", CodePoints: []int{120128}, Characters: []byte{0xf0, 0x9d, 0x95, 0x80}}, + "Iota": {Name: "Iota", CodePoints: []int{921}, Characters: []byte{0xce, 0x99}}, + "Iscr": {Name: "Iscr", CodePoints: []int{8464}, Characters: []byte{0xe2, 0x84, 0x90}}, + "Itilde": {Name: "Itilde", CodePoints: []int{296}, Characters: []byte{0xc4, 0xa8}}, + "Iukcy": {Name: "Iukcy", CodePoints: []int{1030}, Characters: []byte{0xd0, 0x86}}, + "Iuml": {Name: "Iuml", CodePoints: []int{207}, Characters: []byte{0xc3, 0x8f}}, + "Jcirc": {Name: "Jcirc", CodePoints: []int{308}, Characters: []byte{0xc4, 0xb4}}, + "Jcy": {Name: "Jcy", CodePoints: []int{1049}, Characters: []byte{0xd0, 0x99}}, + "Jfr": {Name: "Jfr", CodePoints: []int{120077}, Characters: []byte{0xf0, 0x9d, 0x94, 0x8d}}, + "Jopf": {Name: "Jopf", CodePoints: []int{120129}, Characters: []byte{0xf0, 0x9d, 0x95, 0x81}}, + "Jscr": {Name: "Jscr", CodePoints: []int{119973}, Characters: []byte{0xf0, 0x9d, 0x92, 0xa5}}, + "Jsercy": {Name: "Jsercy", CodePoints: []int{1032}, Characters: []byte{0xd0, 0x88}}, + "Jukcy": {Name: "Jukcy", CodePoints: []int{1028}, Characters: []byte{0xd0, 0x84}}, + "KHcy": {Name: "KHcy", CodePoints: []int{1061}, Characters: []byte{0xd0, 0xa5}}, + "KJcy": {Name: "KJcy", CodePoints: []int{1036}, Characters: []byte{0xd0, 0x8c}}, + "Kappa": {Name: "Kappa", CodePoints: []int{922}, Characters: []byte{0xce, 0x9a}}, + "Kcedil": {Name: "Kcedil", CodePoints: []int{310}, Characters: []byte{0xc4, 0xb6}}, + "Kcy": {Name: "Kcy", CodePoints: []int{1050}, Characters: []byte{0xd0, 0x9a}}, + "Kfr": {Name: "Kfr", CodePoints: []int{120078}, Characters: []byte{0xf0, 0x9d, 0x94, 0x8e}}, + "Kopf": {Name: "Kopf", CodePoints: []int{120130}, Characters: []byte{0xf0, 0x9d, 0x95, 0x82}}, + "Kscr": {Name: "Kscr", CodePoints: []int{119974}, Characters: []byte{0xf0, 0x9d, 0x92, 0xa6}}, + "LJcy": {Name: "LJcy", CodePoints: []int{1033}, Characters: []byte{0xd0, 0x89}}, + "LT": {Name: "LT", CodePoints: []int{60}, Characters: []byte{0x3c}}, + "Lacute": {Name: "Lacute", CodePoints: []int{313}, Characters: []byte{0xc4, 0xb9}}, + "Lambda": {Name: "Lambda", CodePoints: []int{923}, Characters: []byte{0xce, 0x9b}}, + "Lang": {Name: "Lang", CodePoints: []int{10218}, Characters: []byte{0xe2, 0x9f, 0xaa}}, + "Laplacetrf": {Name: "Laplacetrf", CodePoints: []int{8466}, Characters: []byte{0xe2, 0x84, 0x92}}, + "Larr": {Name: "Larr", CodePoints: []int{8606}, Characters: []byte{0xe2, 0x86, 0x9e}}, + "Lcaron": {Name: "Lcaron", CodePoints: []int{317}, Characters: []byte{0xc4, 0xbd}}, + "Lcedil": {Name: "Lcedil", CodePoints: []int{315}, Characters: []byte{0xc4, 0xbb}}, + "Lcy": {Name: "Lcy", CodePoints: []int{1051}, Characters: []byte{0xd0, 0x9b}}, + "LeftAngleBracket": {Name: "LeftAngleBracket", CodePoints: []int{10216}, Characters: []byte{0xe2, 0x9f, 0xa8}}, + "LeftArrow": {Name: "LeftArrow", CodePoints: []int{8592}, Characters: []byte{0xe2, 0x86, 0x90}}, + "LeftArrowBar": {Name: "LeftArrowBar", CodePoints: []int{8676}, Characters: []byte{0xe2, 0x87, 0xa4}}, + "LeftArrowRightArrow": {Name: "LeftArrowRightArrow", CodePoints: []int{8646}, Characters: []byte{0xe2, 0x87, 0x86}}, + "LeftCeiling": {Name: "LeftCeiling", CodePoints: []int{8968}, Characters: []byte{0xe2, 0x8c, 0x88}}, + "LeftDoubleBracket": {Name: "LeftDoubleBracket", CodePoints: []int{10214}, Characters: []byte{0xe2, 0x9f, 0xa6}}, + "LeftDownTeeVector": {Name: "LeftDownTeeVector", CodePoints: []int{10593}, Characters: []byte{0xe2, 0xa5, 0xa1}}, + "LeftDownVector": {Name: "LeftDownVector", CodePoints: []int{8643}, Characters: []byte{0xe2, 0x87, 0x83}}, + "LeftDownVectorBar": {Name: "LeftDownVectorBar", CodePoints: []int{10585}, Characters: []byte{0xe2, 0xa5, 0x99}}, + "LeftFloor": {Name: "LeftFloor", CodePoints: []int{8970}, Characters: []byte{0xe2, 0x8c, 0x8a}}, + "LeftRightArrow": {Name: "LeftRightArrow", CodePoints: []int{8596}, Characters: []byte{0xe2, 0x86, 0x94}}, + "LeftRightVector": {Name: "LeftRightVector", CodePoints: []int{10574}, Characters: []byte{0xe2, 0xa5, 0x8e}}, + "LeftTee": {Name: "LeftTee", CodePoints: []int{8867}, Characters: []byte{0xe2, 0x8a, 0xa3}}, + "LeftTeeArrow": {Name: "LeftTeeArrow", CodePoints: []int{8612}, Characters: []byte{0xe2, 0x86, 0xa4}}, + "LeftTeeVector": {Name: "LeftTeeVector", CodePoints: []int{10586}, Characters: []byte{0xe2, 0xa5, 0x9a}}, + "LeftTriangle": {Name: "LeftTriangle", CodePoints: []int{8882}, Characters: []byte{0xe2, 0x8a, 0xb2}}, + "LeftTriangleBar": {Name: "LeftTriangleBar", CodePoints: []int{10703}, Characters: []byte{0xe2, 0xa7, 0x8f}}, + "LeftTriangleEqual": {Name: "LeftTriangleEqual", CodePoints: []int{8884}, Characters: []byte{0xe2, 0x8a, 0xb4}}, + "LeftUpDownVector": {Name: "LeftUpDownVector", CodePoints: []int{10577}, Characters: []byte{0xe2, 0xa5, 0x91}}, + "LeftUpTeeVector": {Name: "LeftUpTeeVector", CodePoints: []int{10592}, Characters: []byte{0xe2, 0xa5, 0xa0}}, + "LeftUpVector": {Name: "LeftUpVector", CodePoints: []int{8639}, Characters: []byte{0xe2, 0x86, 0xbf}}, + "LeftUpVectorBar": {Name: "LeftUpVectorBar", CodePoints: []int{10584}, Characters: []byte{0xe2, 0xa5, 0x98}}, + "LeftVector": {Name: "LeftVector", CodePoints: []int{8636}, Characters: []byte{0xe2, 0x86, 0xbc}}, + "LeftVectorBar": {Name: "LeftVectorBar", CodePoints: []int{10578}, Characters: []byte{0xe2, 0xa5, 0x92}}, + "Leftarrow": {Name: "Leftarrow", CodePoints: []int{8656}, Characters: []byte{0xe2, 0x87, 0x90}}, + "Leftrightarrow": {Name: "Leftrightarrow", CodePoints: []int{8660}, Characters: []byte{0xe2, 0x87, 0x94}}, + "LessEqualGreater": {Name: "LessEqualGreater", CodePoints: []int{8922}, Characters: []byte{0xe2, 0x8b, 0x9a}}, + "LessFullEqual": {Name: "LessFullEqual", CodePoints: []int{8806}, Characters: []byte{0xe2, 0x89, 0xa6}}, + "LessGreater": {Name: "LessGreater", CodePoints: []int{8822}, Characters: []byte{0xe2, 0x89, 0xb6}}, + "LessLess": {Name: "LessLess", CodePoints: []int{10913}, Characters: []byte{0xe2, 0xaa, 0xa1}}, + "LessSlantEqual": {Name: "LessSlantEqual", CodePoints: []int{10877}, Characters: []byte{0xe2, 0xa9, 0xbd}}, + "LessTilde": {Name: "LessTilde", CodePoints: []int{8818}, Characters: []byte{0xe2, 0x89, 0xb2}}, + "Lfr": {Name: "Lfr", CodePoints: []int{120079}, Characters: []byte{0xf0, 0x9d, 0x94, 0x8f}}, + "Ll": {Name: "Ll", CodePoints: []int{8920}, Characters: []byte{0xe2, 0x8b, 0x98}}, + "Lleftarrow": {Name: "Lleftarrow", CodePoints: []int{8666}, Characters: []byte{0xe2, 0x87, 0x9a}}, + "Lmidot": {Name: "Lmidot", CodePoints: []int{319}, Characters: []byte{0xc4, 0xbf}}, + "LongLeftArrow": {Name: "LongLeftArrow", CodePoints: []int{10229}, Characters: []byte{0xe2, 0x9f, 0xb5}}, + "LongLeftRightArrow": {Name: "LongLeftRightArrow", CodePoints: []int{10231}, Characters: []byte{0xe2, 0x9f, 0xb7}}, + "LongRightArrow": {Name: "LongRightArrow", CodePoints: []int{10230}, Characters: []byte{0xe2, 0x9f, 0xb6}}, + "Longleftarrow": {Name: "Longleftarrow", CodePoints: []int{10232}, Characters: []byte{0xe2, 0x9f, 0xb8}}, + "Longleftrightarrow": {Name: "Longleftrightarrow", CodePoints: []int{10234}, Characters: []byte{0xe2, 0x9f, 0xba}}, + "Longrightarrow": {Name: "Longrightarrow", CodePoints: []int{10233}, Characters: []byte{0xe2, 0x9f, 0xb9}}, + "Lopf": {Name: "Lopf", CodePoints: []int{120131}, Characters: []byte{0xf0, 0x9d, 0x95, 0x83}}, + "LowerLeftArrow": {Name: "LowerLeftArrow", CodePoints: []int{8601}, Characters: []byte{0xe2, 0x86, 0x99}}, + "LowerRightArrow": {Name: "LowerRightArrow", CodePoints: []int{8600}, Characters: []byte{0xe2, 0x86, 0x98}}, + "Lscr": {Name: "Lscr", CodePoints: []int{8466}, Characters: []byte{0xe2, 0x84, 0x92}}, + "Lsh": {Name: "Lsh", CodePoints: []int{8624}, Characters: []byte{0xe2, 0x86, 0xb0}}, + "Lstrok": {Name: "Lstrok", CodePoints: []int{321}, Characters: []byte{0xc5, 0x81}}, + "Lt": {Name: "Lt", CodePoints: []int{8810}, Characters: []byte{0xe2, 0x89, 0xaa}}, + "Map": {Name: "Map", CodePoints: []int{10501}, Characters: []byte{0xe2, 0xa4, 0x85}}, + "Mcy": {Name: "Mcy", CodePoints: []int{1052}, Characters: []byte{0xd0, 0x9c}}, + "MediumSpace": {Name: "MediumSpace", CodePoints: []int{8287}, Characters: []byte{0xe2, 0x81, 0x9f}}, + "Mellintrf": {Name: "Mellintrf", CodePoints: []int{8499}, Characters: []byte{0xe2, 0x84, 0xb3}}, + "Mfr": {Name: "Mfr", CodePoints: []int{120080}, Characters: []byte{0xf0, 0x9d, 0x94, 0x90}}, + "MinusPlus": {Name: "MinusPlus", CodePoints: []int{8723}, Characters: []byte{0xe2, 0x88, 0x93}}, + "Mopf": {Name: "Mopf", CodePoints: []int{120132}, Characters: []byte{0xf0, 0x9d, 0x95, 0x84}}, + "Mscr": {Name: "Mscr", CodePoints: []int{8499}, Characters: []byte{0xe2, 0x84, 0xb3}}, + "Mu": {Name: "Mu", CodePoints: []int{924}, Characters: []byte{0xce, 0x9c}}, + "NJcy": {Name: "NJcy", CodePoints: []int{1034}, Characters: []byte{0xd0, 0x8a}}, + "Nacute": {Name: "Nacute", CodePoints: []int{323}, Characters: []byte{0xc5, 0x83}}, + "Ncaron": {Name: "Ncaron", CodePoints: []int{327}, Characters: []byte{0xc5, 0x87}}, + "Ncedil": {Name: "Ncedil", CodePoints: []int{325}, Characters: []byte{0xc5, 0x85}}, + "Ncy": {Name: "Ncy", CodePoints: []int{1053}, Characters: []byte{0xd0, 0x9d}}, + "NegativeMediumSpace": {Name: "NegativeMediumSpace", CodePoints: []int{8203}, Characters: []byte{0xe2, 0x80, 0x8b}}, + "NegativeThickSpace": {Name: "NegativeThickSpace", CodePoints: []int{8203}, Characters: []byte{0xe2, 0x80, 0x8b}}, + "NegativeThinSpace": {Name: "NegativeThinSpace", CodePoints: []int{8203}, Characters: []byte{0xe2, 0x80, 0x8b}}, + "NegativeVeryThinSpace": {Name: "NegativeVeryThinSpace", CodePoints: []int{8203}, Characters: []byte{0xe2, 0x80, 0x8b}}, + "NestedGreaterGreater": {Name: "NestedGreaterGreater", CodePoints: []int{8811}, Characters: []byte{0xe2, 0x89, 0xab}}, + "NestedLessLess": {Name: "NestedLessLess", CodePoints: []int{8810}, Characters: []byte{0xe2, 0x89, 0xaa}}, + "NewLine": {Name: "NewLine", CodePoints: []int{10}, Characters: []byte{0xa}}, + "Nfr": {Name: "Nfr", CodePoints: []int{120081}, Characters: []byte{0xf0, 0x9d, 0x94, 0x91}}, + "NoBreak": {Name: "NoBreak", CodePoints: []int{8288}, Characters: []byte{0xe2, 0x81, 0xa0}}, + "NonBreakingSpace": {Name: "NonBreakingSpace", CodePoints: []int{160}, Characters: []byte{0xc2, 0xa0}}, + "Nopf": {Name: "Nopf", CodePoints: []int{8469}, Characters: []byte{0xe2, 0x84, 0x95}}, + "Not": {Name: "Not", CodePoints: []int{10988}, Characters: []byte{0xe2, 0xab, 0xac}}, + "NotCongruent": {Name: "NotCongruent", CodePoints: []int{8802}, Characters: []byte{0xe2, 0x89, 0xa2}}, + "NotCupCap": {Name: "NotCupCap", CodePoints: []int{8813}, Characters: []byte{0xe2, 0x89, 0xad}}, + "NotDoubleVerticalBar": {Name: "NotDoubleVerticalBar", CodePoints: []int{8742}, Characters: []byte{0xe2, 0x88, 0xa6}}, + "NotElement": {Name: "NotElement", CodePoints: []int{8713}, Characters: []byte{0xe2, 0x88, 0x89}}, + "NotEqual": {Name: "NotEqual", CodePoints: []int{8800}, Characters: []byte{0xe2, 0x89, 0xa0}}, + "NotEqualTilde": {Name: "NotEqualTilde", CodePoints: []int{8770, 824}, Characters: []byte{0xe2, 0x89, 0x82, 0xcc, 0xb8}}, + "NotExists": {Name: "NotExists", CodePoints: []int{8708}, Characters: []byte{0xe2, 0x88, 0x84}}, + "NotGreater": {Name: "NotGreater", CodePoints: []int{8815}, Characters: []byte{0xe2, 0x89, 0xaf}}, + "NotGreaterEqual": {Name: "NotGreaterEqual", CodePoints: []int{8817}, Characters: []byte{0xe2, 0x89, 0xb1}}, + "NotGreaterFullEqual": {Name: "NotGreaterFullEqual", CodePoints: []int{8807, 824}, Characters: []byte{0xe2, 0x89, 0xa7, 0xcc, 0xb8}}, + "NotGreaterGreater": {Name: "NotGreaterGreater", CodePoints: []int{8811, 824}, Characters: []byte{0xe2, 0x89, 0xab, 0xcc, 0xb8}}, + "NotGreaterLess": {Name: "NotGreaterLess", CodePoints: []int{8825}, Characters: []byte{0xe2, 0x89, 0xb9}}, + "NotGreaterSlantEqual": {Name: "NotGreaterSlantEqual", CodePoints: []int{10878, 824}, Characters: []byte{0xe2, 0xa9, 0xbe, 0xcc, 0xb8}}, + "NotGreaterTilde": {Name: "NotGreaterTilde", CodePoints: []int{8821}, Characters: []byte{0xe2, 0x89, 0xb5}}, + "NotHumpDownHump": {Name: "NotHumpDownHump", CodePoints: []int{8782, 824}, Characters: []byte{0xe2, 0x89, 0x8e, 0xcc, 0xb8}}, + "NotHumpEqual": {Name: "NotHumpEqual", CodePoints: []int{8783, 824}, Characters: []byte{0xe2, 0x89, 0x8f, 0xcc, 0xb8}}, + "NotLeftTriangle": {Name: "NotLeftTriangle", CodePoints: []int{8938}, Characters: []byte{0xe2, 0x8b, 0xaa}}, + "NotLeftTriangleBar": {Name: "NotLeftTriangleBar", CodePoints: []int{10703, 824}, Characters: []byte{0xe2, 0xa7, 0x8f, 0xcc, 0xb8}}, + "NotLeftTriangleEqual": {Name: "NotLeftTriangleEqual", CodePoints: []int{8940}, Characters: []byte{0xe2, 0x8b, 0xac}}, + "NotLess": {Name: "NotLess", CodePoints: []int{8814}, Characters: []byte{0xe2, 0x89, 0xae}}, + "NotLessEqual": {Name: "NotLessEqual", CodePoints: []int{8816}, Characters: []byte{0xe2, 0x89, 0xb0}}, + "NotLessGreater": {Name: "NotLessGreater", CodePoints: []int{8824}, Characters: []byte{0xe2, 0x89, 0xb8}}, + "NotLessLess": {Name: "NotLessLess", CodePoints: []int{8810, 824}, Characters: []byte{0xe2, 0x89, 0xaa, 0xcc, 0xb8}}, + "NotLessSlantEqual": {Name: "NotLessSlantEqual", CodePoints: []int{10877, 824}, Characters: []byte{0xe2, 0xa9, 0xbd, 0xcc, 0xb8}}, + "NotLessTilde": {Name: "NotLessTilde", CodePoints: []int{8820}, Characters: []byte{0xe2, 0x89, 0xb4}}, + "NotNestedGreaterGreater": {Name: "NotNestedGreaterGreater", CodePoints: []int{10914, 824}, Characters: []byte{0xe2, 0xaa, 0xa2, 0xcc, 0xb8}}, + "NotNestedLessLess": {Name: "NotNestedLessLess", CodePoints: []int{10913, 824}, Characters: []byte{0xe2, 0xaa, 0xa1, 0xcc, 0xb8}}, + "NotPrecedes": {Name: "NotPrecedes", CodePoints: []int{8832}, Characters: []byte{0xe2, 0x8a, 0x80}}, + "NotPrecedesEqual": {Name: "NotPrecedesEqual", CodePoints: []int{10927, 824}, Characters: []byte{0xe2, 0xaa, 0xaf, 0xcc, 0xb8}}, + "NotPrecedesSlantEqual": {Name: "NotPrecedesSlantEqual", CodePoints: []int{8928}, Characters: []byte{0xe2, 0x8b, 0xa0}}, + "NotReverseElement": {Name: "NotReverseElement", CodePoints: []int{8716}, Characters: []byte{0xe2, 0x88, 0x8c}}, + "NotRightTriangle": {Name: "NotRightTriangle", CodePoints: []int{8939}, Characters: []byte{0xe2, 0x8b, 0xab}}, + "NotRightTriangleBar": {Name: "NotRightTriangleBar", CodePoints: []int{10704, 824}, Characters: []byte{0xe2, 0xa7, 0x90, 0xcc, 0xb8}}, + "NotRightTriangleEqual": {Name: "NotRightTriangleEqual", CodePoints: []int{8941}, Characters: []byte{0xe2, 0x8b, 0xad}}, + "NotSquareSubset": {Name: "NotSquareSubset", CodePoints: []int{8847, 824}, Characters: []byte{0xe2, 0x8a, 0x8f, 0xcc, 0xb8}}, + "NotSquareSubsetEqual": {Name: "NotSquareSubsetEqual", CodePoints: []int{8930}, Characters: []byte{0xe2, 0x8b, 0xa2}}, + "NotSquareSuperset": {Name: "NotSquareSuperset", CodePoints: []int{8848, 824}, Characters: []byte{0xe2, 0x8a, 0x90, 0xcc, 0xb8}}, + "NotSquareSupersetEqual": {Name: "NotSquareSupersetEqual", CodePoints: []int{8931}, Characters: []byte{0xe2, 0x8b, 0xa3}}, + "NotSubset": {Name: "NotSubset", CodePoints: []int{8834, 8402}, Characters: []byte{0xe2, 0x8a, 0x82, 0xe2, 0x83, 0x92}}, + "NotSubsetEqual": {Name: "NotSubsetEqual", CodePoints: []int{8840}, Characters: []byte{0xe2, 0x8a, 0x88}}, + "NotSucceeds": {Name: "NotSucceeds", CodePoints: []int{8833}, Characters: []byte{0xe2, 0x8a, 0x81}}, + "NotSucceedsEqual": {Name: "NotSucceedsEqual", CodePoints: []int{10928, 824}, Characters: []byte{0xe2, 0xaa, 0xb0, 0xcc, 0xb8}}, + "NotSucceedsSlantEqual": {Name: "NotSucceedsSlantEqual", CodePoints: []int{8929}, Characters: []byte{0xe2, 0x8b, 0xa1}}, + "NotSucceedsTilde": {Name: "NotSucceedsTilde", CodePoints: []int{8831, 824}, Characters: []byte{0xe2, 0x89, 0xbf, 0xcc, 0xb8}}, + "NotSuperset": {Name: "NotSuperset", CodePoints: []int{8835, 8402}, Characters: []byte{0xe2, 0x8a, 0x83, 0xe2, 0x83, 0x92}}, + "NotSupersetEqual": {Name: "NotSupersetEqual", CodePoints: []int{8841}, Characters: []byte{0xe2, 0x8a, 0x89}}, + "NotTilde": {Name: "NotTilde", CodePoints: []int{8769}, Characters: []byte{0xe2, 0x89, 0x81}}, + "NotTildeEqual": {Name: "NotTildeEqual", CodePoints: []int{8772}, Characters: []byte{0xe2, 0x89, 0x84}}, + "NotTildeFullEqual": {Name: "NotTildeFullEqual", CodePoints: []int{8775}, Characters: []byte{0xe2, 0x89, 0x87}}, + "NotTildeTilde": {Name: "NotTildeTilde", CodePoints: []int{8777}, Characters: []byte{0xe2, 0x89, 0x89}}, + "NotVerticalBar": {Name: "NotVerticalBar", CodePoints: []int{8740}, Characters: []byte{0xe2, 0x88, 0xa4}}, + "Nscr": {Name: "Nscr", CodePoints: []int{119977}, Characters: []byte{0xf0, 0x9d, 0x92, 0xa9}}, + "Ntilde": {Name: "Ntilde", CodePoints: []int{209}, Characters: []byte{0xc3, 0x91}}, + "Nu": {Name: "Nu", CodePoints: []int{925}, Characters: []byte{0xce, 0x9d}}, + "OElig": {Name: "OElig", CodePoints: []int{338}, Characters: []byte{0xc5, 0x92}}, + "Oacute": {Name: "Oacute", CodePoints: []int{211}, Characters: []byte{0xc3, 0x93}}, + "Ocirc": {Name: "Ocirc", CodePoints: []int{212}, Characters: []byte{0xc3, 0x94}}, + "Ocy": {Name: "Ocy", CodePoints: []int{1054}, Characters: []byte{0xd0, 0x9e}}, + "Odblac": {Name: "Odblac", CodePoints: []int{336}, Characters: []byte{0xc5, 0x90}}, + "Ofr": {Name: "Ofr", CodePoints: []int{120082}, Characters: []byte{0xf0, 0x9d, 0x94, 0x92}}, + "Ograve": {Name: "Ograve", CodePoints: []int{210}, Characters: []byte{0xc3, 0x92}}, + "Omacr": {Name: "Omacr", CodePoints: []int{332}, Characters: []byte{0xc5, 0x8c}}, + "Omega": {Name: "Omega", CodePoints: []int{937}, Characters: []byte{0xce, 0xa9}}, + "Omicron": {Name: "Omicron", CodePoints: []int{927}, Characters: []byte{0xce, 0x9f}}, + "Oopf": {Name: "Oopf", CodePoints: []int{120134}, Characters: []byte{0xf0, 0x9d, 0x95, 0x86}}, + "OpenCurlyDoubleQuote": {Name: "OpenCurlyDoubleQuote", CodePoints: []int{8220}, Characters: []byte{0xe2, 0x80, 0x9c}}, + "OpenCurlyQuote": {Name: "OpenCurlyQuote", CodePoints: []int{8216}, Characters: []byte{0xe2, 0x80, 0x98}}, + "Or": {Name: "Or", CodePoints: []int{10836}, Characters: []byte{0xe2, 0xa9, 0x94}}, + "Oscr": {Name: "Oscr", CodePoints: []int{119978}, Characters: []byte{0xf0, 0x9d, 0x92, 0xaa}}, + "Oslash": {Name: "Oslash", CodePoints: []int{216}, Characters: []byte{0xc3, 0x98}}, + "Otilde": {Name: "Otilde", CodePoints: []int{213}, Characters: []byte{0xc3, 0x95}}, + "Otimes": {Name: "Otimes", CodePoints: []int{10807}, Characters: []byte{0xe2, 0xa8, 0xb7}}, + "Ouml": {Name: "Ouml", CodePoints: []int{214}, Characters: []byte{0xc3, 0x96}}, + "OverBar": {Name: "OverBar", CodePoints: []int{8254}, Characters: []byte{0xe2, 0x80, 0xbe}}, + "OverBrace": {Name: "OverBrace", CodePoints: []int{9182}, Characters: []byte{0xe2, 0x8f, 0x9e}}, + "OverBracket": {Name: "OverBracket", CodePoints: []int{9140}, Characters: []byte{0xe2, 0x8e, 0xb4}}, + "OverParenthesis": {Name: "OverParenthesis", CodePoints: []int{9180}, Characters: []byte{0xe2, 0x8f, 0x9c}}, + "PartialD": {Name: "PartialD", CodePoints: []int{8706}, Characters: []byte{0xe2, 0x88, 0x82}}, + "Pcy": {Name: "Pcy", CodePoints: []int{1055}, Characters: []byte{0xd0, 0x9f}}, + "Pfr": {Name: "Pfr", CodePoints: []int{120083}, Characters: []byte{0xf0, 0x9d, 0x94, 0x93}}, + "Phi": {Name: "Phi", CodePoints: []int{934}, Characters: []byte{0xce, 0xa6}}, + "Pi": {Name: "Pi", CodePoints: []int{928}, Characters: []byte{0xce, 0xa0}}, + "PlusMinus": {Name: "PlusMinus", CodePoints: []int{177}, Characters: []byte{0xc2, 0xb1}}, + "Poincareplane": {Name: "Poincareplane", CodePoints: []int{8460}, Characters: []byte{0xe2, 0x84, 0x8c}}, + "Popf": {Name: "Popf", CodePoints: []int{8473}, Characters: []byte{0xe2, 0x84, 0x99}}, + "Pr": {Name: "Pr", CodePoints: []int{10939}, Characters: []byte{0xe2, 0xaa, 0xbb}}, + "Precedes": {Name: "Precedes", CodePoints: []int{8826}, Characters: []byte{0xe2, 0x89, 0xba}}, + "PrecedesEqual": {Name: "PrecedesEqual", CodePoints: []int{10927}, Characters: []byte{0xe2, 0xaa, 0xaf}}, + "PrecedesSlantEqual": {Name: "PrecedesSlantEqual", CodePoints: []int{8828}, Characters: []byte{0xe2, 0x89, 0xbc}}, + "PrecedesTilde": {Name: "PrecedesTilde", CodePoints: []int{8830}, Characters: []byte{0xe2, 0x89, 0xbe}}, + "Prime": {Name: "Prime", CodePoints: []int{8243}, Characters: []byte{0xe2, 0x80, 0xb3}}, + "Product": {Name: "Product", CodePoints: []int{8719}, Characters: []byte{0xe2, 0x88, 0x8f}}, + "Proportion": {Name: "Proportion", CodePoints: []int{8759}, Characters: []byte{0xe2, 0x88, 0xb7}}, + "Proportional": {Name: "Proportional", CodePoints: []int{8733}, Characters: []byte{0xe2, 0x88, 0x9d}}, + "Pscr": {Name: "Pscr", CodePoints: []int{119979}, Characters: []byte{0xf0, 0x9d, 0x92, 0xab}}, + "Psi": {Name: "Psi", CodePoints: []int{936}, Characters: []byte{0xce, 0xa8}}, + "QUOT": {Name: "QUOT", CodePoints: []int{34}, Characters: []byte{0x22}}, + "Qfr": {Name: "Qfr", CodePoints: []int{120084}, Characters: []byte{0xf0, 0x9d, 0x94, 0x94}}, + "Qopf": {Name: "Qopf", CodePoints: []int{8474}, Characters: []byte{0xe2, 0x84, 0x9a}}, + "Qscr": {Name: "Qscr", CodePoints: []int{119980}, Characters: []byte{0xf0, 0x9d, 0x92, 0xac}}, + "RBarr": {Name: "RBarr", CodePoints: []int{10512}, Characters: []byte{0xe2, 0xa4, 0x90}}, + "REG": {Name: "REG", CodePoints: []int{174}, Characters: []byte{0xc2, 0xae}}, + "Racute": {Name: "Racute", CodePoints: []int{340}, Characters: []byte{0xc5, 0x94}}, + "Rang": {Name: "Rang", CodePoints: []int{10219}, Characters: []byte{0xe2, 0x9f, 0xab}}, + "Rarr": {Name: "Rarr", CodePoints: []int{8608}, Characters: []byte{0xe2, 0x86, 0xa0}}, + "Rarrtl": {Name: "Rarrtl", CodePoints: []int{10518}, Characters: []byte{0xe2, 0xa4, 0x96}}, + "Rcaron": {Name: "Rcaron", CodePoints: []int{344}, Characters: []byte{0xc5, 0x98}}, + "Rcedil": {Name: "Rcedil", CodePoints: []int{342}, Characters: []byte{0xc5, 0x96}}, + "Rcy": {Name: "Rcy", CodePoints: []int{1056}, Characters: []byte{0xd0, 0xa0}}, + "Re": {Name: "Re", CodePoints: []int{8476}, Characters: []byte{0xe2, 0x84, 0x9c}}, + "ReverseElement": {Name: "ReverseElement", CodePoints: []int{8715}, Characters: []byte{0xe2, 0x88, 0x8b}}, + "ReverseEquilibrium": {Name: "ReverseEquilibrium", CodePoints: []int{8651}, Characters: []byte{0xe2, 0x87, 0x8b}}, + "ReverseUpEquilibrium": {Name: "ReverseUpEquilibrium", CodePoints: []int{10607}, Characters: []byte{0xe2, 0xa5, 0xaf}}, + "Rfr": {Name: "Rfr", CodePoints: []int{8476}, Characters: []byte{0xe2, 0x84, 0x9c}}, + "Rho": {Name: "Rho", CodePoints: []int{929}, Characters: []byte{0xce, 0xa1}}, + "RightAngleBracket": {Name: "RightAngleBracket", CodePoints: []int{10217}, Characters: []byte{0xe2, 0x9f, 0xa9}}, + "RightArrow": {Name: "RightArrow", CodePoints: []int{8594}, Characters: []byte{0xe2, 0x86, 0x92}}, + "RightArrowBar": {Name: "RightArrowBar", CodePoints: []int{8677}, Characters: []byte{0xe2, 0x87, 0xa5}}, + "RightArrowLeftArrow": {Name: "RightArrowLeftArrow", CodePoints: []int{8644}, Characters: []byte{0xe2, 0x87, 0x84}}, + "RightCeiling": {Name: "RightCeiling", CodePoints: []int{8969}, Characters: []byte{0xe2, 0x8c, 0x89}}, + "RightDoubleBracket": {Name: "RightDoubleBracket", CodePoints: []int{10215}, Characters: []byte{0xe2, 0x9f, 0xa7}}, + "RightDownTeeVector": {Name: "RightDownTeeVector", CodePoints: []int{10589}, Characters: []byte{0xe2, 0xa5, 0x9d}}, + "RightDownVector": {Name: "RightDownVector", CodePoints: []int{8642}, Characters: []byte{0xe2, 0x87, 0x82}}, + "RightDownVectorBar": {Name: "RightDownVectorBar", CodePoints: []int{10581}, Characters: []byte{0xe2, 0xa5, 0x95}}, + "RightFloor": {Name: "RightFloor", CodePoints: []int{8971}, Characters: []byte{0xe2, 0x8c, 0x8b}}, + "RightTee": {Name: "RightTee", CodePoints: []int{8866}, Characters: []byte{0xe2, 0x8a, 0xa2}}, + "RightTeeArrow": {Name: "RightTeeArrow", CodePoints: []int{8614}, Characters: []byte{0xe2, 0x86, 0xa6}}, + "RightTeeVector": {Name: "RightTeeVector", CodePoints: []int{10587}, Characters: []byte{0xe2, 0xa5, 0x9b}}, + "RightTriangle": {Name: "RightTriangle", CodePoints: []int{8883}, Characters: []byte{0xe2, 0x8a, 0xb3}}, + "RightTriangleBar": {Name: "RightTriangleBar", CodePoints: []int{10704}, Characters: []byte{0xe2, 0xa7, 0x90}}, + "RightTriangleEqual": {Name: "RightTriangleEqual", CodePoints: []int{8885}, Characters: []byte{0xe2, 0x8a, 0xb5}}, + "RightUpDownVector": {Name: "RightUpDownVector", CodePoints: []int{10575}, Characters: []byte{0xe2, 0xa5, 0x8f}}, + "RightUpTeeVector": {Name: "RightUpTeeVector", CodePoints: []int{10588}, Characters: []byte{0xe2, 0xa5, 0x9c}}, + "RightUpVector": {Name: "RightUpVector", CodePoints: []int{8638}, Characters: []byte{0xe2, 0x86, 0xbe}}, + "RightUpVectorBar": {Name: "RightUpVectorBar", CodePoints: []int{10580}, Characters: []byte{0xe2, 0xa5, 0x94}}, + "RightVector": {Name: "RightVector", CodePoints: []int{8640}, Characters: []byte{0xe2, 0x87, 0x80}}, + "RightVectorBar": {Name: "RightVectorBar", CodePoints: []int{10579}, Characters: []byte{0xe2, 0xa5, 0x93}}, + "Rightarrow": {Name: "Rightarrow", CodePoints: []int{8658}, Characters: []byte{0xe2, 0x87, 0x92}}, + "Ropf": {Name: "Ropf", CodePoints: []int{8477}, Characters: []byte{0xe2, 0x84, 0x9d}}, + "RoundImplies": {Name: "RoundImplies", CodePoints: []int{10608}, Characters: []byte{0xe2, 0xa5, 0xb0}}, + "Rrightarrow": {Name: "Rrightarrow", CodePoints: []int{8667}, Characters: []byte{0xe2, 0x87, 0x9b}}, + "Rscr": {Name: "Rscr", CodePoints: []int{8475}, Characters: []byte{0xe2, 0x84, 0x9b}}, + "Rsh": {Name: "Rsh", CodePoints: []int{8625}, Characters: []byte{0xe2, 0x86, 0xb1}}, + "RuleDelayed": {Name: "RuleDelayed", CodePoints: []int{10740}, Characters: []byte{0xe2, 0xa7, 0xb4}}, + "SHCHcy": {Name: "SHCHcy", CodePoints: []int{1065}, Characters: []byte{0xd0, 0xa9}}, + "SHcy": {Name: "SHcy", CodePoints: []int{1064}, Characters: []byte{0xd0, 0xa8}}, + "SOFTcy": {Name: "SOFTcy", CodePoints: []int{1068}, Characters: []byte{0xd0, 0xac}}, + "Sacute": {Name: "Sacute", CodePoints: []int{346}, Characters: []byte{0xc5, 0x9a}}, + "Sc": {Name: "Sc", CodePoints: []int{10940}, Characters: []byte{0xe2, 0xaa, 0xbc}}, + "Scaron": {Name: "Scaron", CodePoints: []int{352}, Characters: []byte{0xc5, 0xa0}}, + "Scedil": {Name: "Scedil", CodePoints: []int{350}, Characters: []byte{0xc5, 0x9e}}, + "Scirc": {Name: "Scirc", CodePoints: []int{348}, Characters: []byte{0xc5, 0x9c}}, + "Scy": {Name: "Scy", CodePoints: []int{1057}, Characters: []byte{0xd0, 0xa1}}, + "Sfr": {Name: "Sfr", CodePoints: []int{120086}, Characters: []byte{0xf0, 0x9d, 0x94, 0x96}}, + "ShortDownArrow": {Name: "ShortDownArrow", CodePoints: []int{8595}, Characters: []byte{0xe2, 0x86, 0x93}}, + "ShortLeftArrow": {Name: "ShortLeftArrow", CodePoints: []int{8592}, Characters: []byte{0xe2, 0x86, 0x90}}, + "ShortRightArrow": {Name: "ShortRightArrow", CodePoints: []int{8594}, Characters: []byte{0xe2, 0x86, 0x92}}, + "ShortUpArrow": {Name: "ShortUpArrow", CodePoints: []int{8593}, Characters: []byte{0xe2, 0x86, 0x91}}, + "Sigma": {Name: "Sigma", CodePoints: []int{931}, Characters: []byte{0xce, 0xa3}}, + "SmallCircle": {Name: "SmallCircle", CodePoints: []int{8728}, Characters: []byte{0xe2, 0x88, 0x98}}, + "Sopf": {Name: "Sopf", CodePoints: []int{120138}, Characters: []byte{0xf0, 0x9d, 0x95, 0x8a}}, + "Sqrt": {Name: "Sqrt", CodePoints: []int{8730}, Characters: []byte{0xe2, 0x88, 0x9a}}, + "Square": {Name: "Square", CodePoints: []int{9633}, Characters: []byte{0xe2, 0x96, 0xa1}}, + "SquareIntersection": {Name: "SquareIntersection", CodePoints: []int{8851}, Characters: []byte{0xe2, 0x8a, 0x93}}, + "SquareSubset": {Name: "SquareSubset", CodePoints: []int{8847}, Characters: []byte{0xe2, 0x8a, 0x8f}}, + "SquareSubsetEqual": {Name: "SquareSubsetEqual", CodePoints: []int{8849}, Characters: []byte{0xe2, 0x8a, 0x91}}, + "SquareSuperset": {Name: "SquareSuperset", CodePoints: []int{8848}, Characters: []byte{0xe2, 0x8a, 0x90}}, + "SquareSupersetEqual": {Name: "SquareSupersetEqual", CodePoints: []int{8850}, Characters: []byte{0xe2, 0x8a, 0x92}}, + "SquareUnion": {Name: "SquareUnion", CodePoints: []int{8852}, Characters: []byte{0xe2, 0x8a, 0x94}}, + "Sscr": {Name: "Sscr", CodePoints: []int{119982}, Characters: []byte{0xf0, 0x9d, 0x92, 0xae}}, + "Star": {Name: "Star", CodePoints: []int{8902}, Characters: []byte{0xe2, 0x8b, 0x86}}, + "Sub": {Name: "Sub", CodePoints: []int{8912}, Characters: []byte{0xe2, 0x8b, 0x90}}, + "Subset": {Name: "Subset", CodePoints: []int{8912}, Characters: []byte{0xe2, 0x8b, 0x90}}, + "SubsetEqual": {Name: "SubsetEqual", CodePoints: []int{8838}, Characters: []byte{0xe2, 0x8a, 0x86}}, + "Succeeds": {Name: "Succeeds", CodePoints: []int{8827}, Characters: []byte{0xe2, 0x89, 0xbb}}, + "SucceedsEqual": {Name: "SucceedsEqual", CodePoints: []int{10928}, Characters: []byte{0xe2, 0xaa, 0xb0}}, + "SucceedsSlantEqual": {Name: "SucceedsSlantEqual", CodePoints: []int{8829}, Characters: []byte{0xe2, 0x89, 0xbd}}, + "SucceedsTilde": {Name: "SucceedsTilde", CodePoints: []int{8831}, Characters: []byte{0xe2, 0x89, 0xbf}}, + "SuchThat": {Name: "SuchThat", CodePoints: []int{8715}, Characters: []byte{0xe2, 0x88, 0x8b}}, + "Sum": {Name: "Sum", CodePoints: []int{8721}, Characters: []byte{0xe2, 0x88, 0x91}}, + "Sup": {Name: "Sup", CodePoints: []int{8913}, Characters: []byte{0xe2, 0x8b, 0x91}}, + "Superset": {Name: "Superset", CodePoints: []int{8835}, Characters: []byte{0xe2, 0x8a, 0x83}}, + "SupersetEqual": {Name: "SupersetEqual", CodePoints: []int{8839}, Characters: []byte{0xe2, 0x8a, 0x87}}, + "Supset": {Name: "Supset", CodePoints: []int{8913}, Characters: []byte{0xe2, 0x8b, 0x91}}, + "THORN": {Name: "THORN", CodePoints: []int{222}, Characters: []byte{0xc3, 0x9e}}, + "TRADE": {Name: "TRADE", CodePoints: []int{8482}, Characters: []byte{0xe2, 0x84, 0xa2}}, + "TSHcy": {Name: "TSHcy", CodePoints: []int{1035}, Characters: []byte{0xd0, 0x8b}}, + "TScy": {Name: "TScy", CodePoints: []int{1062}, Characters: []byte{0xd0, 0xa6}}, + "Tab": {Name: "Tab", CodePoints: []int{9}, Characters: []byte{0x9}}, + "Tau": {Name: "Tau", CodePoints: []int{932}, Characters: []byte{0xce, 0xa4}}, + "Tcaron": {Name: "Tcaron", CodePoints: []int{356}, Characters: []byte{0xc5, 0xa4}}, + "Tcedil": {Name: "Tcedil", CodePoints: []int{354}, Characters: []byte{0xc5, 0xa2}}, + "Tcy": {Name: "Tcy", CodePoints: []int{1058}, Characters: []byte{0xd0, 0xa2}}, + "Tfr": {Name: "Tfr", CodePoints: []int{120087}, Characters: []byte{0xf0, 0x9d, 0x94, 0x97}}, + "Therefore": {Name: "Therefore", CodePoints: []int{8756}, Characters: []byte{0xe2, 0x88, 0xb4}}, + "Theta": {Name: "Theta", CodePoints: []int{920}, Characters: []byte{0xce, 0x98}}, + "ThickSpace": {Name: "ThickSpace", CodePoints: []int{8287, 8202}, Characters: []byte{0xe2, 0x81, 0x9f, 0xe2, 0x80, 0x8a}}, + "ThinSpace": {Name: "ThinSpace", CodePoints: []int{8201}, Characters: []byte{0xe2, 0x80, 0x89}}, + "Tilde": {Name: "Tilde", CodePoints: []int{8764}, Characters: []byte{0xe2, 0x88, 0xbc}}, + "TildeEqual": {Name: "TildeEqual", CodePoints: []int{8771}, Characters: []byte{0xe2, 0x89, 0x83}}, + "TildeFullEqual": {Name: "TildeFullEqual", CodePoints: []int{8773}, Characters: []byte{0xe2, 0x89, 0x85}}, + "TildeTilde": {Name: "TildeTilde", CodePoints: []int{8776}, Characters: []byte{0xe2, 0x89, 0x88}}, + "Topf": {Name: "Topf", CodePoints: []int{120139}, Characters: []byte{0xf0, 0x9d, 0x95, 0x8b}}, + "TripleDot": {Name: "TripleDot", CodePoints: []int{8411}, Characters: []byte{0xe2, 0x83, 0x9b}}, + "Tscr": {Name: "Tscr", CodePoints: []int{119983}, Characters: []byte{0xf0, 0x9d, 0x92, 0xaf}}, + "Tstrok": {Name: "Tstrok", CodePoints: []int{358}, Characters: []byte{0xc5, 0xa6}}, + "Uacute": {Name: "Uacute", CodePoints: []int{218}, Characters: []byte{0xc3, 0x9a}}, + "Uarr": {Name: "Uarr", CodePoints: []int{8607}, Characters: []byte{0xe2, 0x86, 0x9f}}, + "Uarrocir": {Name: "Uarrocir", CodePoints: []int{10569}, Characters: []byte{0xe2, 0xa5, 0x89}}, + "Ubrcy": {Name: "Ubrcy", CodePoints: []int{1038}, Characters: []byte{0xd0, 0x8e}}, + "Ubreve": {Name: "Ubreve", CodePoints: []int{364}, Characters: []byte{0xc5, 0xac}}, + "Ucirc": {Name: "Ucirc", CodePoints: []int{219}, Characters: []byte{0xc3, 0x9b}}, + "Ucy": {Name: "Ucy", CodePoints: []int{1059}, Characters: []byte{0xd0, 0xa3}}, + "Udblac": {Name: "Udblac", CodePoints: []int{368}, Characters: []byte{0xc5, 0xb0}}, + "Ufr": {Name: "Ufr", CodePoints: []int{120088}, Characters: []byte{0xf0, 0x9d, 0x94, 0x98}}, + "Ugrave": {Name: "Ugrave", CodePoints: []int{217}, Characters: []byte{0xc3, 0x99}}, + "Umacr": {Name: "Umacr", CodePoints: []int{362}, Characters: []byte{0xc5, 0xaa}}, + "UnderBar": {Name: "UnderBar", CodePoints: []int{95}, Characters: []byte{0x5f}}, + "UnderBrace": {Name: "UnderBrace", CodePoints: []int{9183}, Characters: []byte{0xe2, 0x8f, 0x9f}}, + "UnderBracket": {Name: "UnderBracket", CodePoints: []int{9141}, Characters: []byte{0xe2, 0x8e, 0xb5}}, + "UnderParenthesis": {Name: "UnderParenthesis", CodePoints: []int{9181}, Characters: []byte{0xe2, 0x8f, 0x9d}}, + "Union": {Name: "Union", CodePoints: []int{8899}, Characters: []byte{0xe2, 0x8b, 0x83}}, + "UnionPlus": {Name: "UnionPlus", CodePoints: []int{8846}, Characters: []byte{0xe2, 0x8a, 0x8e}}, + "Uogon": {Name: "Uogon", CodePoints: []int{370}, Characters: []byte{0xc5, 0xb2}}, + "Uopf": {Name: "Uopf", CodePoints: []int{120140}, Characters: []byte{0xf0, 0x9d, 0x95, 0x8c}}, + "UpArrow": {Name: "UpArrow", CodePoints: []int{8593}, Characters: []byte{0xe2, 0x86, 0x91}}, + "UpArrowBar": {Name: "UpArrowBar", CodePoints: []int{10514}, Characters: []byte{0xe2, 0xa4, 0x92}}, + "UpArrowDownArrow": {Name: "UpArrowDownArrow", CodePoints: []int{8645}, Characters: []byte{0xe2, 0x87, 0x85}}, + "UpDownArrow": {Name: "UpDownArrow", CodePoints: []int{8597}, Characters: []byte{0xe2, 0x86, 0x95}}, + "UpEquilibrium": {Name: "UpEquilibrium", CodePoints: []int{10606}, Characters: []byte{0xe2, 0xa5, 0xae}}, + "UpTee": {Name: "UpTee", CodePoints: []int{8869}, Characters: []byte{0xe2, 0x8a, 0xa5}}, + "UpTeeArrow": {Name: "UpTeeArrow", CodePoints: []int{8613}, Characters: []byte{0xe2, 0x86, 0xa5}}, + "Uparrow": {Name: "Uparrow", CodePoints: []int{8657}, Characters: []byte{0xe2, 0x87, 0x91}}, + "Updownarrow": {Name: "Updownarrow", CodePoints: []int{8661}, Characters: []byte{0xe2, 0x87, 0x95}}, + "UpperLeftArrow": {Name: "UpperLeftArrow", CodePoints: []int{8598}, Characters: []byte{0xe2, 0x86, 0x96}}, + "UpperRightArrow": {Name: "UpperRightArrow", CodePoints: []int{8599}, Characters: []byte{0xe2, 0x86, 0x97}}, + "Upsi": {Name: "Upsi", CodePoints: []int{978}, Characters: []byte{0xcf, 0x92}}, + "Upsilon": {Name: "Upsilon", CodePoints: []int{933}, Characters: []byte{0xce, 0xa5}}, + "Uring": {Name: "Uring", CodePoints: []int{366}, Characters: []byte{0xc5, 0xae}}, + "Uscr": {Name: "Uscr", CodePoints: []int{119984}, Characters: []byte{0xf0, 0x9d, 0x92, 0xb0}}, + "Utilde": {Name: "Utilde", CodePoints: []int{360}, Characters: []byte{0xc5, 0xa8}}, + "Uuml": {Name: "Uuml", CodePoints: []int{220}, Characters: []byte{0xc3, 0x9c}}, + "VDash": {Name: "VDash", CodePoints: []int{8875}, Characters: []byte{0xe2, 0x8a, 0xab}}, + "Vbar": {Name: "Vbar", CodePoints: []int{10987}, Characters: []byte{0xe2, 0xab, 0xab}}, + "Vcy": {Name: "Vcy", CodePoints: []int{1042}, Characters: []byte{0xd0, 0x92}}, + "Vdash": {Name: "Vdash", CodePoints: []int{8873}, Characters: []byte{0xe2, 0x8a, 0xa9}}, + "Vdashl": {Name: "Vdashl", CodePoints: []int{10982}, Characters: []byte{0xe2, 0xab, 0xa6}}, + "Vee": {Name: "Vee", CodePoints: []int{8897}, Characters: []byte{0xe2, 0x8b, 0x81}}, + "Verbar": {Name: "Verbar", CodePoints: []int{8214}, Characters: []byte{0xe2, 0x80, 0x96}}, + "Vert": {Name: "Vert", CodePoints: []int{8214}, Characters: []byte{0xe2, 0x80, 0x96}}, + "VerticalBar": {Name: "VerticalBar", CodePoints: []int{8739}, Characters: []byte{0xe2, 0x88, 0xa3}}, + "VerticalLine": {Name: "VerticalLine", CodePoints: []int{124}, Characters: []byte{0x7c}}, + "VerticalSeparator": {Name: "VerticalSeparator", CodePoints: []int{10072}, Characters: []byte{0xe2, 0x9d, 0x98}}, + "VerticalTilde": {Name: "VerticalTilde", CodePoints: []int{8768}, Characters: []byte{0xe2, 0x89, 0x80}}, + "VeryThinSpace": {Name: "VeryThinSpace", CodePoints: []int{8202}, Characters: []byte{0xe2, 0x80, 0x8a}}, + "Vfr": {Name: "Vfr", CodePoints: []int{120089}, Characters: []byte{0xf0, 0x9d, 0x94, 0x99}}, + "Vopf": {Name: "Vopf", CodePoints: []int{120141}, Characters: []byte{0xf0, 0x9d, 0x95, 0x8d}}, + "Vscr": {Name: "Vscr", CodePoints: []int{119985}, Characters: []byte{0xf0, 0x9d, 0x92, 0xb1}}, + "Vvdash": {Name: "Vvdash", CodePoints: []int{8874}, Characters: []byte{0xe2, 0x8a, 0xaa}}, + "Wcirc": {Name: "Wcirc", CodePoints: []int{372}, Characters: []byte{0xc5, 0xb4}}, + "Wedge": {Name: "Wedge", CodePoints: []int{8896}, Characters: []byte{0xe2, 0x8b, 0x80}}, + "Wfr": {Name: "Wfr", CodePoints: []int{120090}, Characters: []byte{0xf0, 0x9d, 0x94, 0x9a}}, + "Wopf": {Name: "Wopf", CodePoints: []int{120142}, Characters: []byte{0xf0, 0x9d, 0x95, 0x8e}}, + "Wscr": {Name: "Wscr", CodePoints: []int{119986}, Characters: []byte{0xf0, 0x9d, 0x92, 0xb2}}, + "Xfr": {Name: "Xfr", CodePoints: []int{120091}, Characters: []byte{0xf0, 0x9d, 0x94, 0x9b}}, + "Xi": {Name: "Xi", CodePoints: []int{926}, Characters: []byte{0xce, 0x9e}}, + "Xopf": {Name: "Xopf", CodePoints: []int{120143}, Characters: []byte{0xf0, 0x9d, 0x95, 0x8f}}, + "Xscr": {Name: "Xscr", CodePoints: []int{119987}, Characters: []byte{0xf0, 0x9d, 0x92, 0xb3}}, + "YAcy": {Name: "YAcy", CodePoints: []int{1071}, Characters: []byte{0xd0, 0xaf}}, + "YIcy": {Name: "YIcy", CodePoints: []int{1031}, Characters: []byte{0xd0, 0x87}}, + "YUcy": {Name: "YUcy", CodePoints: []int{1070}, Characters: []byte{0xd0, 0xae}}, + "Yacute": {Name: "Yacute", CodePoints: []int{221}, Characters: []byte{0xc3, 0x9d}}, + "Ycirc": {Name: "Ycirc", CodePoints: []int{374}, Characters: []byte{0xc5, 0xb6}}, + "Ycy": {Name: "Ycy", CodePoints: []int{1067}, Characters: []byte{0xd0, 0xab}}, + "Yfr": {Name: "Yfr", CodePoints: []int{120092}, Characters: []byte{0xf0, 0x9d, 0x94, 0x9c}}, + "Yopf": {Name: "Yopf", CodePoints: []int{120144}, Characters: []byte{0xf0, 0x9d, 0x95, 0x90}}, + "Yscr": {Name: "Yscr", CodePoints: []int{119988}, Characters: []byte{0xf0, 0x9d, 0x92, 0xb4}}, + "Yuml": {Name: "Yuml", CodePoints: []int{376}, Characters: []byte{0xc5, 0xb8}}, + "ZHcy": {Name: "ZHcy", CodePoints: []int{1046}, Characters: []byte{0xd0, 0x96}}, + "Zacute": {Name: "Zacute", CodePoints: []int{377}, Characters: []byte{0xc5, 0xb9}}, + "Zcaron": {Name: "Zcaron", CodePoints: []int{381}, Characters: []byte{0xc5, 0xbd}}, + "Zcy": {Name: "Zcy", CodePoints: []int{1047}, Characters: []byte{0xd0, 0x97}}, + "Zdot": {Name: "Zdot", CodePoints: []int{379}, Characters: []byte{0xc5, 0xbb}}, + "ZeroWidthSpace": {Name: "ZeroWidthSpace", CodePoints: []int{8203}, Characters: []byte{0xe2, 0x80, 0x8b}}, + "Zeta": {Name: "Zeta", CodePoints: []int{918}, Characters: []byte{0xce, 0x96}}, + "Zfr": {Name: "Zfr", CodePoints: []int{8488}, Characters: []byte{0xe2, 0x84, 0xa8}}, + "Zopf": {Name: "Zopf", CodePoints: []int{8484}, Characters: []byte{0xe2, 0x84, 0xa4}}, + "Zscr": {Name: "Zscr", CodePoints: []int{119989}, Characters: []byte{0xf0, 0x9d, 0x92, 0xb5}}, + "aacute": {Name: "aacute", CodePoints: []int{225}, Characters: []byte{0xc3, 0xa1}}, + "abreve": {Name: "abreve", CodePoints: []int{259}, Characters: []byte{0xc4, 0x83}}, + "ac": {Name: "ac", CodePoints: []int{8766}, Characters: []byte{0xe2, 0x88, 0xbe}}, + "acE": {Name: "acE", CodePoints: []int{8766, 819}, Characters: []byte{0xe2, 0x88, 0xbe, 0xcc, 0xb3}}, + "acd": {Name: "acd", CodePoints: []int{8767}, Characters: []byte{0xe2, 0x88, 0xbf}}, + "acirc": {Name: "acirc", CodePoints: []int{226}, Characters: []byte{0xc3, 0xa2}}, + "acute": {Name: "acute", CodePoints: []int{180}, Characters: []byte{0xc2, 0xb4}}, + "acy": {Name: "acy", CodePoints: []int{1072}, Characters: []byte{0xd0, 0xb0}}, + "aelig": {Name: "aelig", CodePoints: []int{230}, Characters: []byte{0xc3, 0xa6}}, + "af": {Name: "af", CodePoints: []int{8289}, Characters: []byte{0xe2, 0x81, 0xa1}}, + "afr": {Name: "afr", CodePoints: []int{120094}, Characters: []byte{0xf0, 0x9d, 0x94, 0x9e}}, + "agrave": {Name: "agrave", CodePoints: []int{224}, Characters: []byte{0xc3, 0xa0}}, + "alefsym": {Name: "alefsym", CodePoints: []int{8501}, Characters: []byte{0xe2, 0x84, 0xb5}}, + "aleph": {Name: "aleph", CodePoints: []int{8501}, Characters: []byte{0xe2, 0x84, 0xb5}}, + "alpha": {Name: "alpha", CodePoints: []int{945}, Characters: []byte{0xce, 0xb1}}, + "amacr": {Name: "amacr", CodePoints: []int{257}, Characters: []byte{0xc4, 0x81}}, + "amalg": {Name: "amalg", CodePoints: []int{10815}, Characters: []byte{0xe2, 0xa8, 0xbf}}, + "amp": {Name: "amp", CodePoints: []int{38}, Characters: []byte{0x26}}, + "and": {Name: "and", CodePoints: []int{8743}, Characters: []byte{0xe2, 0x88, 0xa7}}, + "andand": {Name: "andand", CodePoints: []int{10837}, Characters: []byte{0xe2, 0xa9, 0x95}}, + "andd": {Name: "andd", CodePoints: []int{10844}, Characters: []byte{0xe2, 0xa9, 0x9c}}, + "andslope": {Name: "andslope", CodePoints: []int{10840}, Characters: []byte{0xe2, 0xa9, 0x98}}, + "andv": {Name: "andv", CodePoints: []int{10842}, Characters: []byte{0xe2, 0xa9, 0x9a}}, + "ang": {Name: "ang", CodePoints: []int{8736}, Characters: []byte{0xe2, 0x88, 0xa0}}, + "ange": {Name: "ange", CodePoints: []int{10660}, Characters: []byte{0xe2, 0xa6, 0xa4}}, + "angle": {Name: "angle", CodePoints: []int{8736}, Characters: []byte{0xe2, 0x88, 0xa0}}, + "angmsd": {Name: "angmsd", CodePoints: []int{8737}, Characters: []byte{0xe2, 0x88, 0xa1}}, + "angmsdaa": {Name: "angmsdaa", CodePoints: []int{10664}, Characters: []byte{0xe2, 0xa6, 0xa8}}, + "angmsdab": {Name: "angmsdab", CodePoints: []int{10665}, Characters: []byte{0xe2, 0xa6, 0xa9}}, + "angmsdac": {Name: "angmsdac", CodePoints: []int{10666}, Characters: []byte{0xe2, 0xa6, 0xaa}}, + "angmsdad": {Name: "angmsdad", CodePoints: []int{10667}, Characters: []byte{0xe2, 0xa6, 0xab}}, + "angmsdae": {Name: "angmsdae", CodePoints: []int{10668}, Characters: []byte{0xe2, 0xa6, 0xac}}, + "angmsdaf": {Name: "angmsdaf", CodePoints: []int{10669}, Characters: []byte{0xe2, 0xa6, 0xad}}, + "angmsdag": {Name: "angmsdag", CodePoints: []int{10670}, Characters: []byte{0xe2, 0xa6, 0xae}}, + "angmsdah": {Name: "angmsdah", CodePoints: []int{10671}, Characters: []byte{0xe2, 0xa6, 0xaf}}, + "angrt": {Name: "angrt", CodePoints: []int{8735}, Characters: []byte{0xe2, 0x88, 0x9f}}, + "angrtvb": {Name: "angrtvb", CodePoints: []int{8894}, Characters: []byte{0xe2, 0x8a, 0xbe}}, + "angrtvbd": {Name: "angrtvbd", CodePoints: []int{10653}, Characters: []byte{0xe2, 0xa6, 0x9d}}, + "angsph": {Name: "angsph", CodePoints: []int{8738}, Characters: []byte{0xe2, 0x88, 0xa2}}, + "angst": {Name: "angst", CodePoints: []int{197}, Characters: []byte{0xc3, 0x85}}, + "angzarr": {Name: "angzarr", CodePoints: []int{9084}, Characters: []byte{0xe2, 0x8d, 0xbc}}, + "aogon": {Name: "aogon", CodePoints: []int{261}, Characters: []byte{0xc4, 0x85}}, + "aopf": {Name: "aopf", CodePoints: []int{120146}, Characters: []byte{0xf0, 0x9d, 0x95, 0x92}}, + "ap": {Name: "ap", CodePoints: []int{8776}, Characters: []byte{0xe2, 0x89, 0x88}}, + "apE": {Name: "apE", CodePoints: []int{10864}, Characters: []byte{0xe2, 0xa9, 0xb0}}, + "apacir": {Name: "apacir", CodePoints: []int{10863}, Characters: []byte{0xe2, 0xa9, 0xaf}}, + "ape": {Name: "ape", CodePoints: []int{8778}, Characters: []byte{0xe2, 0x89, 0x8a}}, + "apid": {Name: "apid", CodePoints: []int{8779}, Characters: []byte{0xe2, 0x89, 0x8b}}, + "apos": {Name: "apos", CodePoints: []int{39}, Characters: []byte{0x27}}, + "approx": {Name: "approx", CodePoints: []int{8776}, Characters: []byte{0xe2, 0x89, 0x88}}, + "approxeq": {Name: "approxeq", CodePoints: []int{8778}, Characters: []byte{0xe2, 0x89, 0x8a}}, + "aring": {Name: "aring", CodePoints: []int{229}, Characters: []byte{0xc3, 0xa5}}, + "ascr": {Name: "ascr", CodePoints: []int{119990}, Characters: []byte{0xf0, 0x9d, 0x92, 0xb6}}, + "ast": {Name: "ast", CodePoints: []int{42}, Characters: []byte{0x2a}}, + "asymp": {Name: "asymp", CodePoints: []int{8776}, Characters: []byte{0xe2, 0x89, 0x88}}, + "asympeq": {Name: "asympeq", CodePoints: []int{8781}, Characters: []byte{0xe2, 0x89, 0x8d}}, + "atilde": {Name: "atilde", CodePoints: []int{227}, Characters: []byte{0xc3, 0xa3}}, + "auml": {Name: "auml", CodePoints: []int{228}, Characters: []byte{0xc3, 0xa4}}, + "awconint": {Name: "awconint", CodePoints: []int{8755}, Characters: []byte{0xe2, 0x88, 0xb3}}, + "awint": {Name: "awint", CodePoints: []int{10769}, Characters: []byte{0xe2, 0xa8, 0x91}}, + "bNot": {Name: "bNot", CodePoints: []int{10989}, Characters: []byte{0xe2, 0xab, 0xad}}, + "backcong": {Name: "backcong", CodePoints: []int{8780}, Characters: []byte{0xe2, 0x89, 0x8c}}, + "backepsilon": {Name: "backepsilon", CodePoints: []int{1014}, Characters: []byte{0xcf, 0xb6}}, + "backprime": {Name: "backprime", CodePoints: []int{8245}, Characters: []byte{0xe2, 0x80, 0xb5}}, + "backsim": {Name: "backsim", CodePoints: []int{8765}, Characters: []byte{0xe2, 0x88, 0xbd}}, + "backsimeq": {Name: "backsimeq", CodePoints: []int{8909}, Characters: []byte{0xe2, 0x8b, 0x8d}}, + "barvee": {Name: "barvee", CodePoints: []int{8893}, Characters: []byte{0xe2, 0x8a, 0xbd}}, + "barwed": {Name: "barwed", CodePoints: []int{8965}, Characters: []byte{0xe2, 0x8c, 0x85}}, + "barwedge": {Name: "barwedge", CodePoints: []int{8965}, Characters: []byte{0xe2, 0x8c, 0x85}}, + "bbrk": {Name: "bbrk", CodePoints: []int{9141}, Characters: []byte{0xe2, 0x8e, 0xb5}}, + "bbrktbrk": {Name: "bbrktbrk", CodePoints: []int{9142}, Characters: []byte{0xe2, 0x8e, 0xb6}}, + "bcong": {Name: "bcong", CodePoints: []int{8780}, Characters: []byte{0xe2, 0x89, 0x8c}}, + "bcy": {Name: "bcy", CodePoints: []int{1073}, Characters: []byte{0xd0, 0xb1}}, + "bdquo": {Name: "bdquo", CodePoints: []int{8222}, Characters: []byte{0xe2, 0x80, 0x9e}}, + "becaus": {Name: "becaus", CodePoints: []int{8757}, Characters: []byte{0xe2, 0x88, 0xb5}}, + "because": {Name: "because", CodePoints: []int{8757}, Characters: []byte{0xe2, 0x88, 0xb5}}, + "bemptyv": {Name: "bemptyv", CodePoints: []int{10672}, Characters: []byte{0xe2, 0xa6, 0xb0}}, + "bepsi": {Name: "bepsi", CodePoints: []int{1014}, Characters: []byte{0xcf, 0xb6}}, + "bernou": {Name: "bernou", CodePoints: []int{8492}, Characters: []byte{0xe2, 0x84, 0xac}}, + "beta": {Name: "beta", CodePoints: []int{946}, Characters: []byte{0xce, 0xb2}}, + "beth": {Name: "beth", CodePoints: []int{8502}, Characters: []byte{0xe2, 0x84, 0xb6}}, + "between": {Name: "between", CodePoints: []int{8812}, Characters: []byte{0xe2, 0x89, 0xac}}, + "bfr": {Name: "bfr", CodePoints: []int{120095}, Characters: []byte{0xf0, 0x9d, 0x94, 0x9f}}, + "bigcap": {Name: "bigcap", CodePoints: []int{8898}, Characters: []byte{0xe2, 0x8b, 0x82}}, + "bigcirc": {Name: "bigcirc", CodePoints: []int{9711}, Characters: []byte{0xe2, 0x97, 0xaf}}, + "bigcup": {Name: "bigcup", CodePoints: []int{8899}, Characters: []byte{0xe2, 0x8b, 0x83}}, + "bigodot": {Name: "bigodot", CodePoints: []int{10752}, Characters: []byte{0xe2, 0xa8, 0x80}}, + "bigoplus": {Name: "bigoplus", CodePoints: []int{10753}, Characters: []byte{0xe2, 0xa8, 0x81}}, + "bigotimes": {Name: "bigotimes", CodePoints: []int{10754}, Characters: []byte{0xe2, 0xa8, 0x82}}, + "bigsqcup": {Name: "bigsqcup", CodePoints: []int{10758}, Characters: []byte{0xe2, 0xa8, 0x86}}, + "bigstar": {Name: "bigstar", CodePoints: []int{9733}, Characters: []byte{0xe2, 0x98, 0x85}}, + "bigtriangledown": {Name: "bigtriangledown", CodePoints: []int{9661}, Characters: []byte{0xe2, 0x96, 0xbd}}, + "bigtriangleup": {Name: "bigtriangleup", CodePoints: []int{9651}, Characters: []byte{0xe2, 0x96, 0xb3}}, + "biguplus": {Name: "biguplus", CodePoints: []int{10756}, Characters: []byte{0xe2, 0xa8, 0x84}}, + "bigvee": {Name: "bigvee", CodePoints: []int{8897}, Characters: []byte{0xe2, 0x8b, 0x81}}, + "bigwedge": {Name: "bigwedge", CodePoints: []int{8896}, Characters: []byte{0xe2, 0x8b, 0x80}}, + "bkarow": {Name: "bkarow", CodePoints: []int{10509}, Characters: []byte{0xe2, 0xa4, 0x8d}}, + "blacklozenge": {Name: "blacklozenge", CodePoints: []int{10731}, Characters: []byte{0xe2, 0xa7, 0xab}}, + "blacksquare": {Name: "blacksquare", CodePoints: []int{9642}, Characters: []byte{0xe2, 0x96, 0xaa}}, + "blacktriangle": {Name: "blacktriangle", CodePoints: []int{9652}, Characters: []byte{0xe2, 0x96, 0xb4}}, + "blacktriangledown": {Name: "blacktriangledown", CodePoints: []int{9662}, Characters: []byte{0xe2, 0x96, 0xbe}}, + "blacktriangleleft": {Name: "blacktriangleleft", CodePoints: []int{9666}, Characters: []byte{0xe2, 0x97, 0x82}}, + "blacktriangleright": {Name: "blacktriangleright", CodePoints: []int{9656}, Characters: []byte{0xe2, 0x96, 0xb8}}, + "blank": {Name: "blank", CodePoints: []int{9251}, Characters: []byte{0xe2, 0x90, 0xa3}}, + "blk12": {Name: "blk12", CodePoints: []int{9618}, Characters: []byte{0xe2, 0x96, 0x92}}, + "blk14": {Name: "blk14", CodePoints: []int{9617}, Characters: []byte{0xe2, 0x96, 0x91}}, + "blk34": {Name: "blk34", CodePoints: []int{9619}, Characters: []byte{0xe2, 0x96, 0x93}}, + "block": {Name: "block", CodePoints: []int{9608}, Characters: []byte{0xe2, 0x96, 0x88}}, + "bne": {Name: "bne", CodePoints: []int{61, 8421}, Characters: []byte{0x3d, 0xe2, 0x83, 0xa5}}, + "bnequiv": {Name: "bnequiv", CodePoints: []int{8801, 8421}, Characters: []byte{0xe2, 0x89, 0xa1, 0xe2, 0x83, 0xa5}}, + "bnot": {Name: "bnot", CodePoints: []int{8976}, Characters: []byte{0xe2, 0x8c, 0x90}}, + "bopf": {Name: "bopf", CodePoints: []int{120147}, Characters: []byte{0xf0, 0x9d, 0x95, 0x93}}, + "bot": {Name: "bot", CodePoints: []int{8869}, Characters: []byte{0xe2, 0x8a, 0xa5}}, + "bottom": {Name: "bottom", CodePoints: []int{8869}, Characters: []byte{0xe2, 0x8a, 0xa5}}, + "bowtie": {Name: "bowtie", CodePoints: []int{8904}, Characters: []byte{0xe2, 0x8b, 0x88}}, + "boxDL": {Name: "boxDL", CodePoints: []int{9559}, Characters: []byte{0xe2, 0x95, 0x97}}, + "boxDR": {Name: "boxDR", CodePoints: []int{9556}, Characters: []byte{0xe2, 0x95, 0x94}}, + "boxDl": {Name: "boxDl", CodePoints: []int{9558}, Characters: []byte{0xe2, 0x95, 0x96}}, + "boxDr": {Name: "boxDr", CodePoints: []int{9555}, Characters: []byte{0xe2, 0x95, 0x93}}, + "boxH": {Name: "boxH", CodePoints: []int{9552}, Characters: []byte{0xe2, 0x95, 0x90}}, + "boxHD": {Name: "boxHD", CodePoints: []int{9574}, Characters: []byte{0xe2, 0x95, 0xa6}}, + "boxHU": {Name: "boxHU", CodePoints: []int{9577}, Characters: []byte{0xe2, 0x95, 0xa9}}, + "boxHd": {Name: "boxHd", CodePoints: []int{9572}, Characters: []byte{0xe2, 0x95, 0xa4}}, + "boxHu": {Name: "boxHu", CodePoints: []int{9575}, Characters: []byte{0xe2, 0x95, 0xa7}}, + "boxUL": {Name: "boxUL", CodePoints: []int{9565}, Characters: []byte{0xe2, 0x95, 0x9d}}, + "boxUR": {Name: "boxUR", CodePoints: []int{9562}, Characters: []byte{0xe2, 0x95, 0x9a}}, + "boxUl": {Name: "boxUl", CodePoints: []int{9564}, Characters: []byte{0xe2, 0x95, 0x9c}}, + "boxUr": {Name: "boxUr", CodePoints: []int{9561}, Characters: []byte{0xe2, 0x95, 0x99}}, + "boxV": {Name: "boxV", CodePoints: []int{9553}, Characters: []byte{0xe2, 0x95, 0x91}}, + "boxVH": {Name: "boxVH", CodePoints: []int{9580}, Characters: []byte{0xe2, 0x95, 0xac}}, + "boxVL": {Name: "boxVL", CodePoints: []int{9571}, Characters: []byte{0xe2, 0x95, 0xa3}}, + "boxVR": {Name: "boxVR", CodePoints: []int{9568}, Characters: []byte{0xe2, 0x95, 0xa0}}, + "boxVh": {Name: "boxVh", CodePoints: []int{9579}, Characters: []byte{0xe2, 0x95, 0xab}}, + "boxVl": {Name: "boxVl", CodePoints: []int{9570}, Characters: []byte{0xe2, 0x95, 0xa2}}, + "boxVr": {Name: "boxVr", CodePoints: []int{9567}, Characters: []byte{0xe2, 0x95, 0x9f}}, + "boxbox": {Name: "boxbox", CodePoints: []int{10697}, Characters: []byte{0xe2, 0xa7, 0x89}}, + "boxdL": {Name: "boxdL", CodePoints: []int{9557}, Characters: []byte{0xe2, 0x95, 0x95}}, + "boxdR": {Name: "boxdR", CodePoints: []int{9554}, Characters: []byte{0xe2, 0x95, 0x92}}, + "boxdl": {Name: "boxdl", CodePoints: []int{9488}, Characters: []byte{0xe2, 0x94, 0x90}}, + "boxdr": {Name: "boxdr", CodePoints: []int{9484}, Characters: []byte{0xe2, 0x94, 0x8c}}, + "boxh": {Name: "boxh", CodePoints: []int{9472}, Characters: []byte{0xe2, 0x94, 0x80}}, + "boxhD": {Name: "boxhD", CodePoints: []int{9573}, Characters: []byte{0xe2, 0x95, 0xa5}}, + "boxhU": {Name: "boxhU", CodePoints: []int{9576}, Characters: []byte{0xe2, 0x95, 0xa8}}, + "boxhd": {Name: "boxhd", CodePoints: []int{9516}, Characters: []byte{0xe2, 0x94, 0xac}}, + "boxhu": {Name: "boxhu", CodePoints: []int{9524}, Characters: []byte{0xe2, 0x94, 0xb4}}, + "boxminus": {Name: "boxminus", CodePoints: []int{8863}, Characters: []byte{0xe2, 0x8a, 0x9f}}, + "boxplus": {Name: "boxplus", CodePoints: []int{8862}, Characters: []byte{0xe2, 0x8a, 0x9e}}, + "boxtimes": {Name: "boxtimes", CodePoints: []int{8864}, Characters: []byte{0xe2, 0x8a, 0xa0}}, + "boxuL": {Name: "boxuL", CodePoints: []int{9563}, Characters: []byte{0xe2, 0x95, 0x9b}}, + "boxuR": {Name: "boxuR", CodePoints: []int{9560}, Characters: []byte{0xe2, 0x95, 0x98}}, + "boxul": {Name: "boxul", CodePoints: []int{9496}, Characters: []byte{0xe2, 0x94, 0x98}}, + "boxur": {Name: "boxur", CodePoints: []int{9492}, Characters: []byte{0xe2, 0x94, 0x94}}, + "boxv": {Name: "boxv", CodePoints: []int{9474}, Characters: []byte{0xe2, 0x94, 0x82}}, + "boxvH": {Name: "boxvH", CodePoints: []int{9578}, Characters: []byte{0xe2, 0x95, 0xaa}}, + "boxvL": {Name: "boxvL", CodePoints: []int{9569}, Characters: []byte{0xe2, 0x95, 0xa1}}, + "boxvR": {Name: "boxvR", CodePoints: []int{9566}, Characters: []byte{0xe2, 0x95, 0x9e}}, + "boxvh": {Name: "boxvh", CodePoints: []int{9532}, Characters: []byte{0xe2, 0x94, 0xbc}}, + "boxvl": {Name: "boxvl", CodePoints: []int{9508}, Characters: []byte{0xe2, 0x94, 0xa4}}, + "boxvr": {Name: "boxvr", CodePoints: []int{9500}, Characters: []byte{0xe2, 0x94, 0x9c}}, + "bprime": {Name: "bprime", CodePoints: []int{8245}, Characters: []byte{0xe2, 0x80, 0xb5}}, + "breve": {Name: "breve", CodePoints: []int{728}, Characters: []byte{0xcb, 0x98}}, + "brvbar": {Name: "brvbar", CodePoints: []int{166}, Characters: []byte{0xc2, 0xa6}}, + "bscr": {Name: "bscr", CodePoints: []int{119991}, Characters: []byte{0xf0, 0x9d, 0x92, 0xb7}}, + "bsemi": {Name: "bsemi", CodePoints: []int{8271}, Characters: []byte{0xe2, 0x81, 0x8f}}, + "bsim": {Name: "bsim", CodePoints: []int{8765}, Characters: []byte{0xe2, 0x88, 0xbd}}, + "bsime": {Name: "bsime", CodePoints: []int{8909}, Characters: []byte{0xe2, 0x8b, 0x8d}}, + "bsol": {Name: "bsol", CodePoints: []int{92}, Characters: []byte{0x5c}}, + "bsolb": {Name: "bsolb", CodePoints: []int{10693}, Characters: []byte{0xe2, 0xa7, 0x85}}, + "bsolhsub": {Name: "bsolhsub", CodePoints: []int{10184}, Characters: []byte{0xe2, 0x9f, 0x88}}, + "bull": {Name: "bull", CodePoints: []int{8226}, Characters: []byte{0xe2, 0x80, 0xa2}}, + "bullet": {Name: "bullet", CodePoints: []int{8226}, Characters: []byte{0xe2, 0x80, 0xa2}}, + "bump": {Name: "bump", CodePoints: []int{8782}, Characters: []byte{0xe2, 0x89, 0x8e}}, + "bumpE": {Name: "bumpE", CodePoints: []int{10926}, Characters: []byte{0xe2, 0xaa, 0xae}}, + "bumpe": {Name: "bumpe", CodePoints: []int{8783}, Characters: []byte{0xe2, 0x89, 0x8f}}, + "bumpeq": {Name: "bumpeq", CodePoints: []int{8783}, Characters: []byte{0xe2, 0x89, 0x8f}}, + "cacute": {Name: "cacute", CodePoints: []int{263}, Characters: []byte{0xc4, 0x87}}, + "cap": {Name: "cap", CodePoints: []int{8745}, Characters: []byte{0xe2, 0x88, 0xa9}}, + "capand": {Name: "capand", CodePoints: []int{10820}, Characters: []byte{0xe2, 0xa9, 0x84}}, + "capbrcup": {Name: "capbrcup", CodePoints: []int{10825}, Characters: []byte{0xe2, 0xa9, 0x89}}, + "capcap": {Name: "capcap", CodePoints: []int{10827}, Characters: []byte{0xe2, 0xa9, 0x8b}}, + "capcup": {Name: "capcup", CodePoints: []int{10823}, Characters: []byte{0xe2, 0xa9, 0x87}}, + "capdot": {Name: "capdot", CodePoints: []int{10816}, Characters: []byte{0xe2, 0xa9, 0x80}}, + "caps": {Name: "caps", CodePoints: []int{8745, 65024}, Characters: []byte{0xe2, 0x88, 0xa9, 0xef, 0xb8, 0x80}}, + "caret": {Name: "caret", CodePoints: []int{8257}, Characters: []byte{0xe2, 0x81, 0x81}}, + "caron": {Name: "caron", CodePoints: []int{711}, Characters: []byte{0xcb, 0x87}}, + "ccaps": {Name: "ccaps", CodePoints: []int{10829}, Characters: []byte{0xe2, 0xa9, 0x8d}}, + "ccaron": {Name: "ccaron", CodePoints: []int{269}, Characters: []byte{0xc4, 0x8d}}, + "ccedil": {Name: "ccedil", CodePoints: []int{231}, Characters: []byte{0xc3, 0xa7}}, + "ccirc": {Name: "ccirc", CodePoints: []int{265}, Characters: []byte{0xc4, 0x89}}, + "ccups": {Name: "ccups", CodePoints: []int{10828}, Characters: []byte{0xe2, 0xa9, 0x8c}}, + "ccupssm": {Name: "ccupssm", CodePoints: []int{10832}, Characters: []byte{0xe2, 0xa9, 0x90}}, + "cdot": {Name: "cdot", CodePoints: []int{267}, Characters: []byte{0xc4, 0x8b}}, + "cedil": {Name: "cedil", CodePoints: []int{184}, Characters: []byte{0xc2, 0xb8}}, + "cemptyv": {Name: "cemptyv", CodePoints: []int{10674}, Characters: []byte{0xe2, 0xa6, 0xb2}}, + "cent": {Name: "cent", CodePoints: []int{162}, Characters: []byte{0xc2, 0xa2}}, + "centerdot": {Name: "centerdot", CodePoints: []int{183}, Characters: []byte{0xc2, 0xb7}}, + "cfr": {Name: "cfr", CodePoints: []int{120096}, Characters: []byte{0xf0, 0x9d, 0x94, 0xa0}}, + "chcy": {Name: "chcy", CodePoints: []int{1095}, Characters: []byte{0xd1, 0x87}}, + "check": {Name: "check", CodePoints: []int{10003}, Characters: []byte{0xe2, 0x9c, 0x93}}, + "checkmark": {Name: "checkmark", CodePoints: []int{10003}, Characters: []byte{0xe2, 0x9c, 0x93}}, + "chi": {Name: "chi", CodePoints: []int{967}, Characters: []byte{0xcf, 0x87}}, + "cir": {Name: "cir", CodePoints: []int{9675}, Characters: []byte{0xe2, 0x97, 0x8b}}, + "cirE": {Name: "cirE", CodePoints: []int{10691}, Characters: []byte{0xe2, 0xa7, 0x83}}, + "circ": {Name: "circ", CodePoints: []int{710}, Characters: []byte{0xcb, 0x86}}, + "circeq": {Name: "circeq", CodePoints: []int{8791}, Characters: []byte{0xe2, 0x89, 0x97}}, + "circlearrowleft": {Name: "circlearrowleft", CodePoints: []int{8634}, Characters: []byte{0xe2, 0x86, 0xba}}, + "circlearrowright": {Name: "circlearrowright", CodePoints: []int{8635}, Characters: []byte{0xe2, 0x86, 0xbb}}, + "circledR": {Name: "circledR", CodePoints: []int{174}, Characters: []byte{0xc2, 0xae}}, + "circledS": {Name: "circledS", CodePoints: []int{9416}, Characters: []byte{0xe2, 0x93, 0x88}}, + "circledast": {Name: "circledast", CodePoints: []int{8859}, Characters: []byte{0xe2, 0x8a, 0x9b}}, + "circledcirc": {Name: "circledcirc", CodePoints: []int{8858}, Characters: []byte{0xe2, 0x8a, 0x9a}}, + "circleddash": {Name: "circleddash", CodePoints: []int{8861}, Characters: []byte{0xe2, 0x8a, 0x9d}}, + "cire": {Name: "cire", CodePoints: []int{8791}, Characters: []byte{0xe2, 0x89, 0x97}}, + "cirfnint": {Name: "cirfnint", CodePoints: []int{10768}, Characters: []byte{0xe2, 0xa8, 0x90}}, + "cirmid": {Name: "cirmid", CodePoints: []int{10991}, Characters: []byte{0xe2, 0xab, 0xaf}}, + "cirscir": {Name: "cirscir", CodePoints: []int{10690}, Characters: []byte{0xe2, 0xa7, 0x82}}, + "clubs": {Name: "clubs", CodePoints: []int{9827}, Characters: []byte{0xe2, 0x99, 0xa3}}, + "clubsuit": {Name: "clubsuit", CodePoints: []int{9827}, Characters: []byte{0xe2, 0x99, 0xa3}}, + "colon": {Name: "colon", CodePoints: []int{58}, Characters: []byte{0x3a}}, + "colone": {Name: "colone", CodePoints: []int{8788}, Characters: []byte{0xe2, 0x89, 0x94}}, + "coloneq": {Name: "coloneq", CodePoints: []int{8788}, Characters: []byte{0xe2, 0x89, 0x94}}, + "comma": {Name: "comma", CodePoints: []int{44}, Characters: []byte{0x2c}}, + "commat": {Name: "commat", CodePoints: []int{64}, Characters: []byte{0x40}}, + "comp": {Name: "comp", CodePoints: []int{8705}, Characters: []byte{0xe2, 0x88, 0x81}}, + "compfn": {Name: "compfn", CodePoints: []int{8728}, Characters: []byte{0xe2, 0x88, 0x98}}, + "complement": {Name: "complement", CodePoints: []int{8705}, Characters: []byte{0xe2, 0x88, 0x81}}, + "complexes": {Name: "complexes", CodePoints: []int{8450}, Characters: []byte{0xe2, 0x84, 0x82}}, + "cong": {Name: "cong", CodePoints: []int{8773}, Characters: []byte{0xe2, 0x89, 0x85}}, + "congdot": {Name: "congdot", CodePoints: []int{10861}, Characters: []byte{0xe2, 0xa9, 0xad}}, + "conint": {Name: "conint", CodePoints: []int{8750}, Characters: []byte{0xe2, 0x88, 0xae}}, + "copf": {Name: "copf", CodePoints: []int{120148}, Characters: []byte{0xf0, 0x9d, 0x95, 0x94}}, + "coprod": {Name: "coprod", CodePoints: []int{8720}, Characters: []byte{0xe2, 0x88, 0x90}}, + "copy": {Name: "copy", CodePoints: []int{169}, Characters: []byte{0xc2, 0xa9}}, + "copysr": {Name: "copysr", CodePoints: []int{8471}, Characters: []byte{0xe2, 0x84, 0x97}}, + "crarr": {Name: "crarr", CodePoints: []int{8629}, Characters: []byte{0xe2, 0x86, 0xb5}}, + "cross": {Name: "cross", CodePoints: []int{10007}, Characters: []byte{0xe2, 0x9c, 0x97}}, + "cscr": {Name: "cscr", CodePoints: []int{119992}, Characters: []byte{0xf0, 0x9d, 0x92, 0xb8}}, + "csub": {Name: "csub", CodePoints: []int{10959}, Characters: []byte{0xe2, 0xab, 0x8f}}, + "csube": {Name: "csube", CodePoints: []int{10961}, Characters: []byte{0xe2, 0xab, 0x91}}, + "csup": {Name: "csup", CodePoints: []int{10960}, Characters: []byte{0xe2, 0xab, 0x90}}, + "csupe": {Name: "csupe", CodePoints: []int{10962}, Characters: []byte{0xe2, 0xab, 0x92}}, + "ctdot": {Name: "ctdot", CodePoints: []int{8943}, Characters: []byte{0xe2, 0x8b, 0xaf}}, + "cudarrl": {Name: "cudarrl", CodePoints: []int{10552}, Characters: []byte{0xe2, 0xa4, 0xb8}}, + "cudarrr": {Name: "cudarrr", CodePoints: []int{10549}, Characters: []byte{0xe2, 0xa4, 0xb5}}, + "cuepr": {Name: "cuepr", CodePoints: []int{8926}, Characters: []byte{0xe2, 0x8b, 0x9e}}, + "cuesc": {Name: "cuesc", CodePoints: []int{8927}, Characters: []byte{0xe2, 0x8b, 0x9f}}, + "cularr": {Name: "cularr", CodePoints: []int{8630}, Characters: []byte{0xe2, 0x86, 0xb6}}, + "cularrp": {Name: "cularrp", CodePoints: []int{10557}, Characters: []byte{0xe2, 0xa4, 0xbd}}, + "cup": {Name: "cup", CodePoints: []int{8746}, Characters: []byte{0xe2, 0x88, 0xaa}}, + "cupbrcap": {Name: "cupbrcap", CodePoints: []int{10824}, Characters: []byte{0xe2, 0xa9, 0x88}}, + "cupcap": {Name: "cupcap", CodePoints: []int{10822}, Characters: []byte{0xe2, 0xa9, 0x86}}, + "cupcup": {Name: "cupcup", CodePoints: []int{10826}, Characters: []byte{0xe2, 0xa9, 0x8a}}, + "cupdot": {Name: "cupdot", CodePoints: []int{8845}, Characters: []byte{0xe2, 0x8a, 0x8d}}, + "cupor": {Name: "cupor", CodePoints: []int{10821}, Characters: []byte{0xe2, 0xa9, 0x85}}, + "cups": {Name: "cups", CodePoints: []int{8746, 65024}, Characters: []byte{0xe2, 0x88, 0xaa, 0xef, 0xb8, 0x80}}, + "curarr": {Name: "curarr", CodePoints: []int{8631}, Characters: []byte{0xe2, 0x86, 0xb7}}, + "curarrm": {Name: "curarrm", CodePoints: []int{10556}, Characters: []byte{0xe2, 0xa4, 0xbc}}, + "curlyeqprec": {Name: "curlyeqprec", CodePoints: []int{8926}, Characters: []byte{0xe2, 0x8b, 0x9e}}, + "curlyeqsucc": {Name: "curlyeqsucc", CodePoints: []int{8927}, Characters: []byte{0xe2, 0x8b, 0x9f}}, + "curlyvee": {Name: "curlyvee", CodePoints: []int{8910}, Characters: []byte{0xe2, 0x8b, 0x8e}}, + "curlywedge": {Name: "curlywedge", CodePoints: []int{8911}, Characters: []byte{0xe2, 0x8b, 0x8f}}, + "curren": {Name: "curren", CodePoints: []int{164}, Characters: []byte{0xc2, 0xa4}}, + "curvearrowleft": {Name: "curvearrowleft", CodePoints: []int{8630}, Characters: []byte{0xe2, 0x86, 0xb6}}, + "curvearrowright": {Name: "curvearrowright", CodePoints: []int{8631}, Characters: []byte{0xe2, 0x86, 0xb7}}, + "cuvee": {Name: "cuvee", CodePoints: []int{8910}, Characters: []byte{0xe2, 0x8b, 0x8e}}, + "cuwed": {Name: "cuwed", CodePoints: []int{8911}, Characters: []byte{0xe2, 0x8b, 0x8f}}, + "cwconint": {Name: "cwconint", CodePoints: []int{8754}, Characters: []byte{0xe2, 0x88, 0xb2}}, + "cwint": {Name: "cwint", CodePoints: []int{8753}, Characters: []byte{0xe2, 0x88, 0xb1}}, + "cylcty": {Name: "cylcty", CodePoints: []int{9005}, Characters: []byte{0xe2, 0x8c, 0xad}}, + "dArr": {Name: "dArr", CodePoints: []int{8659}, Characters: []byte{0xe2, 0x87, 0x93}}, + "dHar": {Name: "dHar", CodePoints: []int{10597}, Characters: []byte{0xe2, 0xa5, 0xa5}}, + "dagger": {Name: "dagger", CodePoints: []int{8224}, Characters: []byte{0xe2, 0x80, 0xa0}}, + "daleth": {Name: "daleth", CodePoints: []int{8504}, Characters: []byte{0xe2, 0x84, 0xb8}}, + "darr": {Name: "darr", CodePoints: []int{8595}, Characters: []byte{0xe2, 0x86, 0x93}}, + "dash": {Name: "dash", CodePoints: []int{8208}, Characters: []byte{0xe2, 0x80, 0x90}}, + "dashv": {Name: "dashv", CodePoints: []int{8867}, Characters: []byte{0xe2, 0x8a, 0xa3}}, + "dbkarow": {Name: "dbkarow", CodePoints: []int{10511}, Characters: []byte{0xe2, 0xa4, 0x8f}}, + "dblac": {Name: "dblac", CodePoints: []int{733}, Characters: []byte{0xcb, 0x9d}}, + "dcaron": {Name: "dcaron", CodePoints: []int{271}, Characters: []byte{0xc4, 0x8f}}, + "dcy": {Name: "dcy", CodePoints: []int{1076}, Characters: []byte{0xd0, 0xb4}}, + "dd": {Name: "dd", CodePoints: []int{8518}, Characters: []byte{0xe2, 0x85, 0x86}}, + "ddagger": {Name: "ddagger", CodePoints: []int{8225}, Characters: []byte{0xe2, 0x80, 0xa1}}, + "ddarr": {Name: "ddarr", CodePoints: []int{8650}, Characters: []byte{0xe2, 0x87, 0x8a}}, + "ddotseq": {Name: "ddotseq", CodePoints: []int{10871}, Characters: []byte{0xe2, 0xa9, 0xb7}}, + "deg": {Name: "deg", CodePoints: []int{176}, Characters: []byte{0xc2, 0xb0}}, + "delta": {Name: "delta", CodePoints: []int{948}, Characters: []byte{0xce, 0xb4}}, + "demptyv": {Name: "demptyv", CodePoints: []int{10673}, Characters: []byte{0xe2, 0xa6, 0xb1}}, + "dfisht": {Name: "dfisht", CodePoints: []int{10623}, Characters: []byte{0xe2, 0xa5, 0xbf}}, + "dfr": {Name: "dfr", CodePoints: []int{120097}, Characters: []byte{0xf0, 0x9d, 0x94, 0xa1}}, + "dharl": {Name: "dharl", CodePoints: []int{8643}, Characters: []byte{0xe2, 0x87, 0x83}}, + "dharr": {Name: "dharr", CodePoints: []int{8642}, Characters: []byte{0xe2, 0x87, 0x82}}, + "diam": {Name: "diam", CodePoints: []int{8900}, Characters: []byte{0xe2, 0x8b, 0x84}}, + "diamond": {Name: "diamond", CodePoints: []int{8900}, Characters: []byte{0xe2, 0x8b, 0x84}}, + "diamondsuit": {Name: "diamondsuit", CodePoints: []int{9830}, Characters: []byte{0xe2, 0x99, 0xa6}}, + "diams": {Name: "diams", CodePoints: []int{9830}, Characters: []byte{0xe2, 0x99, 0xa6}}, + "die": {Name: "die", CodePoints: []int{168}, Characters: []byte{0xc2, 0xa8}}, + "digamma": {Name: "digamma", CodePoints: []int{989}, Characters: []byte{0xcf, 0x9d}}, + "disin": {Name: "disin", CodePoints: []int{8946}, Characters: []byte{0xe2, 0x8b, 0xb2}}, + "div": {Name: "div", CodePoints: []int{247}, Characters: []byte{0xc3, 0xb7}}, + "divide": {Name: "divide", CodePoints: []int{247}, Characters: []byte{0xc3, 0xb7}}, + "divideontimes": {Name: "divideontimes", CodePoints: []int{8903}, Characters: []byte{0xe2, 0x8b, 0x87}}, + "divonx": {Name: "divonx", CodePoints: []int{8903}, Characters: []byte{0xe2, 0x8b, 0x87}}, + "djcy": {Name: "djcy", CodePoints: []int{1106}, Characters: []byte{0xd1, 0x92}}, + "dlcorn": {Name: "dlcorn", CodePoints: []int{8990}, Characters: []byte{0xe2, 0x8c, 0x9e}}, + "dlcrop": {Name: "dlcrop", CodePoints: []int{8973}, Characters: []byte{0xe2, 0x8c, 0x8d}}, + "dollar": {Name: "dollar", CodePoints: []int{36}, Characters: []byte{0x24}}, + "dopf": {Name: "dopf", CodePoints: []int{120149}, Characters: []byte{0xf0, 0x9d, 0x95, 0x95}}, + "dot": {Name: "dot", CodePoints: []int{729}, Characters: []byte{0xcb, 0x99}}, + "doteq": {Name: "doteq", CodePoints: []int{8784}, Characters: []byte{0xe2, 0x89, 0x90}}, + "doteqdot": {Name: "doteqdot", CodePoints: []int{8785}, Characters: []byte{0xe2, 0x89, 0x91}}, + "dotminus": {Name: "dotminus", CodePoints: []int{8760}, Characters: []byte{0xe2, 0x88, 0xb8}}, + "dotplus": {Name: "dotplus", CodePoints: []int{8724}, Characters: []byte{0xe2, 0x88, 0x94}}, + "dotsquare": {Name: "dotsquare", CodePoints: []int{8865}, Characters: []byte{0xe2, 0x8a, 0xa1}}, + "doublebarwedge": {Name: "doublebarwedge", CodePoints: []int{8966}, Characters: []byte{0xe2, 0x8c, 0x86}}, + "downarrow": {Name: "downarrow", CodePoints: []int{8595}, Characters: []byte{0xe2, 0x86, 0x93}}, + "downdownarrows": {Name: "downdownarrows", CodePoints: []int{8650}, Characters: []byte{0xe2, 0x87, 0x8a}}, + "downharpoonleft": {Name: "downharpoonleft", CodePoints: []int{8643}, Characters: []byte{0xe2, 0x87, 0x83}}, + "downharpoonright": {Name: "downharpoonright", CodePoints: []int{8642}, Characters: []byte{0xe2, 0x87, 0x82}}, + "drbkarow": {Name: "drbkarow", CodePoints: []int{10512}, Characters: []byte{0xe2, 0xa4, 0x90}}, + "drcorn": {Name: "drcorn", CodePoints: []int{8991}, Characters: []byte{0xe2, 0x8c, 0x9f}}, + "drcrop": {Name: "drcrop", CodePoints: []int{8972}, Characters: []byte{0xe2, 0x8c, 0x8c}}, + "dscr": {Name: "dscr", CodePoints: []int{119993}, Characters: []byte{0xf0, 0x9d, 0x92, 0xb9}}, + "dscy": {Name: "dscy", CodePoints: []int{1109}, Characters: []byte{0xd1, 0x95}}, + "dsol": {Name: "dsol", CodePoints: []int{10742}, Characters: []byte{0xe2, 0xa7, 0xb6}}, + "dstrok": {Name: "dstrok", CodePoints: []int{273}, Characters: []byte{0xc4, 0x91}}, + "dtdot": {Name: "dtdot", CodePoints: []int{8945}, Characters: []byte{0xe2, 0x8b, 0xb1}}, + "dtri": {Name: "dtri", CodePoints: []int{9663}, Characters: []byte{0xe2, 0x96, 0xbf}}, + "dtrif": {Name: "dtrif", CodePoints: []int{9662}, Characters: []byte{0xe2, 0x96, 0xbe}}, + "duarr": {Name: "duarr", CodePoints: []int{8693}, Characters: []byte{0xe2, 0x87, 0xb5}}, + "duhar": {Name: "duhar", CodePoints: []int{10607}, Characters: []byte{0xe2, 0xa5, 0xaf}}, + "dwangle": {Name: "dwangle", CodePoints: []int{10662}, Characters: []byte{0xe2, 0xa6, 0xa6}}, + "dzcy": {Name: "dzcy", CodePoints: []int{1119}, Characters: []byte{0xd1, 0x9f}}, + "dzigrarr": {Name: "dzigrarr", CodePoints: []int{10239}, Characters: []byte{0xe2, 0x9f, 0xbf}}, + "eDDot": {Name: "eDDot", CodePoints: []int{10871}, Characters: []byte{0xe2, 0xa9, 0xb7}}, + "eDot": {Name: "eDot", CodePoints: []int{8785}, Characters: []byte{0xe2, 0x89, 0x91}}, + "eacute": {Name: "eacute", CodePoints: []int{233}, Characters: []byte{0xc3, 0xa9}}, + "easter": {Name: "easter", CodePoints: []int{10862}, Characters: []byte{0xe2, 0xa9, 0xae}}, + "ecaron": {Name: "ecaron", CodePoints: []int{283}, Characters: []byte{0xc4, 0x9b}}, + "ecir": {Name: "ecir", CodePoints: []int{8790}, Characters: []byte{0xe2, 0x89, 0x96}}, + "ecirc": {Name: "ecirc", CodePoints: []int{234}, Characters: []byte{0xc3, 0xaa}}, + "ecolon": {Name: "ecolon", CodePoints: []int{8789}, Characters: []byte{0xe2, 0x89, 0x95}}, + "ecy": {Name: "ecy", CodePoints: []int{1101}, Characters: []byte{0xd1, 0x8d}}, + "edot": {Name: "edot", CodePoints: []int{279}, Characters: []byte{0xc4, 0x97}}, + "ee": {Name: "ee", CodePoints: []int{8519}, Characters: []byte{0xe2, 0x85, 0x87}}, + "efDot": {Name: "efDot", CodePoints: []int{8786}, Characters: []byte{0xe2, 0x89, 0x92}}, + "efr": {Name: "efr", CodePoints: []int{120098}, Characters: []byte{0xf0, 0x9d, 0x94, 0xa2}}, + "eg": {Name: "eg", CodePoints: []int{10906}, Characters: []byte{0xe2, 0xaa, 0x9a}}, + "egrave": {Name: "egrave", CodePoints: []int{232}, Characters: []byte{0xc3, 0xa8}}, + "egs": {Name: "egs", CodePoints: []int{10902}, Characters: []byte{0xe2, 0xaa, 0x96}}, + "egsdot": {Name: "egsdot", CodePoints: []int{10904}, Characters: []byte{0xe2, 0xaa, 0x98}}, + "el": {Name: "el", CodePoints: []int{10905}, Characters: []byte{0xe2, 0xaa, 0x99}}, + "elinters": {Name: "elinters", CodePoints: []int{9191}, Characters: []byte{0xe2, 0x8f, 0xa7}}, + "ell": {Name: "ell", CodePoints: []int{8467}, Characters: []byte{0xe2, 0x84, 0x93}}, + "els": {Name: "els", CodePoints: []int{10901}, Characters: []byte{0xe2, 0xaa, 0x95}}, + "elsdot": {Name: "elsdot", CodePoints: []int{10903}, Characters: []byte{0xe2, 0xaa, 0x97}}, + "emacr": {Name: "emacr", CodePoints: []int{275}, Characters: []byte{0xc4, 0x93}}, + "empty": {Name: "empty", CodePoints: []int{8709}, Characters: []byte{0xe2, 0x88, 0x85}}, + "emptyset": {Name: "emptyset", CodePoints: []int{8709}, Characters: []byte{0xe2, 0x88, 0x85}}, + "emptyv": {Name: "emptyv", CodePoints: []int{8709}, Characters: []byte{0xe2, 0x88, 0x85}}, + "emsp": {Name: "emsp", CodePoints: []int{8195}, Characters: []byte{0xe2, 0x80, 0x83}}, + "emsp13": {Name: "emsp13", CodePoints: []int{8196}, Characters: []byte{0xe2, 0x80, 0x84}}, + "emsp14": {Name: "emsp14", CodePoints: []int{8197}, Characters: []byte{0xe2, 0x80, 0x85}}, + "eng": {Name: "eng", CodePoints: []int{331}, Characters: []byte{0xc5, 0x8b}}, + "ensp": {Name: "ensp", CodePoints: []int{8194}, Characters: []byte{0xe2, 0x80, 0x82}}, + "eogon": {Name: "eogon", CodePoints: []int{281}, Characters: []byte{0xc4, 0x99}}, + "eopf": {Name: "eopf", CodePoints: []int{120150}, Characters: []byte{0xf0, 0x9d, 0x95, 0x96}}, + "epar": {Name: "epar", CodePoints: []int{8917}, Characters: []byte{0xe2, 0x8b, 0x95}}, + "eparsl": {Name: "eparsl", CodePoints: []int{10723}, Characters: []byte{0xe2, 0xa7, 0xa3}}, + "eplus": {Name: "eplus", CodePoints: []int{10865}, Characters: []byte{0xe2, 0xa9, 0xb1}}, + "epsi": {Name: "epsi", CodePoints: []int{949}, Characters: []byte{0xce, 0xb5}}, + "epsilon": {Name: "epsilon", CodePoints: []int{949}, Characters: []byte{0xce, 0xb5}}, + "epsiv": {Name: "epsiv", CodePoints: []int{1013}, Characters: []byte{0xcf, 0xb5}}, + "eqcirc": {Name: "eqcirc", CodePoints: []int{8790}, Characters: []byte{0xe2, 0x89, 0x96}}, + "eqcolon": {Name: "eqcolon", CodePoints: []int{8789}, Characters: []byte{0xe2, 0x89, 0x95}}, + "eqsim": {Name: "eqsim", CodePoints: []int{8770}, Characters: []byte{0xe2, 0x89, 0x82}}, + "eqslantgtr": {Name: "eqslantgtr", CodePoints: []int{10902}, Characters: []byte{0xe2, 0xaa, 0x96}}, + "eqslantless": {Name: "eqslantless", CodePoints: []int{10901}, Characters: []byte{0xe2, 0xaa, 0x95}}, + "equals": {Name: "equals", CodePoints: []int{61}, Characters: []byte{0x3d}}, + "equest": {Name: "equest", CodePoints: []int{8799}, Characters: []byte{0xe2, 0x89, 0x9f}}, + "equiv": {Name: "equiv", CodePoints: []int{8801}, Characters: []byte{0xe2, 0x89, 0xa1}}, + "equivDD": {Name: "equivDD", CodePoints: []int{10872}, Characters: []byte{0xe2, 0xa9, 0xb8}}, + "eqvparsl": {Name: "eqvparsl", CodePoints: []int{10725}, Characters: []byte{0xe2, 0xa7, 0xa5}}, + "erDot": {Name: "erDot", CodePoints: []int{8787}, Characters: []byte{0xe2, 0x89, 0x93}}, + "erarr": {Name: "erarr", CodePoints: []int{10609}, Characters: []byte{0xe2, 0xa5, 0xb1}}, + "escr": {Name: "escr", CodePoints: []int{8495}, Characters: []byte{0xe2, 0x84, 0xaf}}, + "esdot": {Name: "esdot", CodePoints: []int{8784}, Characters: []byte{0xe2, 0x89, 0x90}}, + "esim": {Name: "esim", CodePoints: []int{8770}, Characters: []byte{0xe2, 0x89, 0x82}}, + "eta": {Name: "eta", CodePoints: []int{951}, Characters: []byte{0xce, 0xb7}}, + "eth": {Name: "eth", CodePoints: []int{240}, Characters: []byte{0xc3, 0xb0}}, + "euml": {Name: "euml", CodePoints: []int{235}, Characters: []byte{0xc3, 0xab}}, + "euro": {Name: "euro", CodePoints: []int{8364}, Characters: []byte{0xe2, 0x82, 0xac}}, + "excl": {Name: "excl", CodePoints: []int{33}, Characters: []byte{0x21}}, + "exist": {Name: "exist", CodePoints: []int{8707}, Characters: []byte{0xe2, 0x88, 0x83}}, + "expectation": {Name: "expectation", CodePoints: []int{8496}, Characters: []byte{0xe2, 0x84, 0xb0}}, + "exponentiale": {Name: "exponentiale", CodePoints: []int{8519}, Characters: []byte{0xe2, 0x85, 0x87}}, + "fallingdotseq": {Name: "fallingdotseq", CodePoints: []int{8786}, Characters: []byte{0xe2, 0x89, 0x92}}, + "fcy": {Name: "fcy", CodePoints: []int{1092}, Characters: []byte{0xd1, 0x84}}, + "female": {Name: "female", CodePoints: []int{9792}, Characters: []byte{0xe2, 0x99, 0x80}}, + "ffilig": {Name: "ffilig", CodePoints: []int{64259}, Characters: []byte{0xef, 0xac, 0x83}}, + "fflig": {Name: "fflig", CodePoints: []int{64256}, Characters: []byte{0xef, 0xac, 0x80}}, + "ffllig": {Name: "ffllig", CodePoints: []int{64260}, Characters: []byte{0xef, 0xac, 0x84}}, + "ffr": {Name: "ffr", CodePoints: []int{120099}, Characters: []byte{0xf0, 0x9d, 0x94, 0xa3}}, + "filig": {Name: "filig", CodePoints: []int{64257}, Characters: []byte{0xef, 0xac, 0x81}}, + "fjlig": {Name: "fjlig", CodePoints: []int{102, 106}, Characters: []byte{0x66, 0x6a}}, + "flat": {Name: "flat", CodePoints: []int{9837}, Characters: []byte{0xe2, 0x99, 0xad}}, + "fllig": {Name: "fllig", CodePoints: []int{64258}, Characters: []byte{0xef, 0xac, 0x82}}, + "fltns": {Name: "fltns", CodePoints: []int{9649}, Characters: []byte{0xe2, 0x96, 0xb1}}, + "fnof": {Name: "fnof", CodePoints: []int{402}, Characters: []byte{0xc6, 0x92}}, + "fopf": {Name: "fopf", CodePoints: []int{120151}, Characters: []byte{0xf0, 0x9d, 0x95, 0x97}}, + "forall": {Name: "forall", CodePoints: []int{8704}, Characters: []byte{0xe2, 0x88, 0x80}}, + "fork": {Name: "fork", CodePoints: []int{8916}, Characters: []byte{0xe2, 0x8b, 0x94}}, + "forkv": {Name: "forkv", CodePoints: []int{10969}, Characters: []byte{0xe2, 0xab, 0x99}}, + "fpartint": {Name: "fpartint", CodePoints: []int{10765}, Characters: []byte{0xe2, 0xa8, 0x8d}}, + "frac12": {Name: "frac12", CodePoints: []int{189}, Characters: []byte{0xc2, 0xbd}}, + "frac13": {Name: "frac13", CodePoints: []int{8531}, Characters: []byte{0xe2, 0x85, 0x93}}, + "frac14": {Name: "frac14", CodePoints: []int{188}, Characters: []byte{0xc2, 0xbc}}, + "frac15": {Name: "frac15", CodePoints: []int{8533}, Characters: []byte{0xe2, 0x85, 0x95}}, + "frac16": {Name: "frac16", CodePoints: []int{8537}, Characters: []byte{0xe2, 0x85, 0x99}}, + "frac18": {Name: "frac18", CodePoints: []int{8539}, Characters: []byte{0xe2, 0x85, 0x9b}}, + "frac23": {Name: "frac23", CodePoints: []int{8532}, Characters: []byte{0xe2, 0x85, 0x94}}, + "frac25": {Name: "frac25", CodePoints: []int{8534}, Characters: []byte{0xe2, 0x85, 0x96}}, + "frac34": {Name: "frac34", CodePoints: []int{190}, Characters: []byte{0xc2, 0xbe}}, + "frac35": {Name: "frac35", CodePoints: []int{8535}, Characters: []byte{0xe2, 0x85, 0x97}}, + "frac38": {Name: "frac38", CodePoints: []int{8540}, Characters: []byte{0xe2, 0x85, 0x9c}}, + "frac45": {Name: "frac45", CodePoints: []int{8536}, Characters: []byte{0xe2, 0x85, 0x98}}, + "frac56": {Name: "frac56", CodePoints: []int{8538}, Characters: []byte{0xe2, 0x85, 0x9a}}, + "frac58": {Name: "frac58", CodePoints: []int{8541}, Characters: []byte{0xe2, 0x85, 0x9d}}, + "frac78": {Name: "frac78", CodePoints: []int{8542}, Characters: []byte{0xe2, 0x85, 0x9e}}, + "frasl": {Name: "frasl", CodePoints: []int{8260}, Characters: []byte{0xe2, 0x81, 0x84}}, + "frown": {Name: "frown", CodePoints: []int{8994}, Characters: []byte{0xe2, 0x8c, 0xa2}}, + "fscr": {Name: "fscr", CodePoints: []int{119995}, Characters: []byte{0xf0, 0x9d, 0x92, 0xbb}}, + "gE": {Name: "gE", CodePoints: []int{8807}, Characters: []byte{0xe2, 0x89, 0xa7}}, + "gEl": {Name: "gEl", CodePoints: []int{10892}, Characters: []byte{0xe2, 0xaa, 0x8c}}, + "gacute": {Name: "gacute", CodePoints: []int{501}, Characters: []byte{0xc7, 0xb5}}, + "gamma": {Name: "gamma", CodePoints: []int{947}, Characters: []byte{0xce, 0xb3}}, + "gammad": {Name: "gammad", CodePoints: []int{989}, Characters: []byte{0xcf, 0x9d}}, + "gap": {Name: "gap", CodePoints: []int{10886}, Characters: []byte{0xe2, 0xaa, 0x86}}, + "gbreve": {Name: "gbreve", CodePoints: []int{287}, Characters: []byte{0xc4, 0x9f}}, + "gcirc": {Name: "gcirc", CodePoints: []int{285}, Characters: []byte{0xc4, 0x9d}}, + "gcy": {Name: "gcy", CodePoints: []int{1075}, Characters: []byte{0xd0, 0xb3}}, + "gdot": {Name: "gdot", CodePoints: []int{289}, Characters: []byte{0xc4, 0xa1}}, + "ge": {Name: "ge", CodePoints: []int{8805}, Characters: []byte{0xe2, 0x89, 0xa5}}, + "gel": {Name: "gel", CodePoints: []int{8923}, Characters: []byte{0xe2, 0x8b, 0x9b}}, + "geq": {Name: "geq", CodePoints: []int{8805}, Characters: []byte{0xe2, 0x89, 0xa5}}, + "geqq": {Name: "geqq", CodePoints: []int{8807}, Characters: []byte{0xe2, 0x89, 0xa7}}, + "geqslant": {Name: "geqslant", CodePoints: []int{10878}, Characters: []byte{0xe2, 0xa9, 0xbe}}, + "ges": {Name: "ges", CodePoints: []int{10878}, Characters: []byte{0xe2, 0xa9, 0xbe}}, + "gescc": {Name: "gescc", CodePoints: []int{10921}, Characters: []byte{0xe2, 0xaa, 0xa9}}, + "gesdot": {Name: "gesdot", CodePoints: []int{10880}, Characters: []byte{0xe2, 0xaa, 0x80}}, + "gesdoto": {Name: "gesdoto", CodePoints: []int{10882}, Characters: []byte{0xe2, 0xaa, 0x82}}, + "gesdotol": {Name: "gesdotol", CodePoints: []int{10884}, Characters: []byte{0xe2, 0xaa, 0x84}}, + "gesl": {Name: "gesl", CodePoints: []int{8923, 65024}, Characters: []byte{0xe2, 0x8b, 0x9b, 0xef, 0xb8, 0x80}}, + "gesles": {Name: "gesles", CodePoints: []int{10900}, Characters: []byte{0xe2, 0xaa, 0x94}}, + "gfr": {Name: "gfr", CodePoints: []int{120100}, Characters: []byte{0xf0, 0x9d, 0x94, 0xa4}}, + "gg": {Name: "gg", CodePoints: []int{8811}, Characters: []byte{0xe2, 0x89, 0xab}}, + "ggg": {Name: "ggg", CodePoints: []int{8921}, Characters: []byte{0xe2, 0x8b, 0x99}}, + "gimel": {Name: "gimel", CodePoints: []int{8503}, Characters: []byte{0xe2, 0x84, 0xb7}}, + "gjcy": {Name: "gjcy", CodePoints: []int{1107}, Characters: []byte{0xd1, 0x93}}, + "gl": {Name: "gl", CodePoints: []int{8823}, Characters: []byte{0xe2, 0x89, 0xb7}}, + "glE": {Name: "glE", CodePoints: []int{10898}, Characters: []byte{0xe2, 0xaa, 0x92}}, + "gla": {Name: "gla", CodePoints: []int{10917}, Characters: []byte{0xe2, 0xaa, 0xa5}}, + "glj": {Name: "glj", CodePoints: []int{10916}, Characters: []byte{0xe2, 0xaa, 0xa4}}, + "gnE": {Name: "gnE", CodePoints: []int{8809}, Characters: []byte{0xe2, 0x89, 0xa9}}, + "gnap": {Name: "gnap", CodePoints: []int{10890}, Characters: []byte{0xe2, 0xaa, 0x8a}}, + "gnapprox": {Name: "gnapprox", CodePoints: []int{10890}, Characters: []byte{0xe2, 0xaa, 0x8a}}, + "gne": {Name: "gne", CodePoints: []int{10888}, Characters: []byte{0xe2, 0xaa, 0x88}}, + "gneq": {Name: "gneq", CodePoints: []int{10888}, Characters: []byte{0xe2, 0xaa, 0x88}}, + "gneqq": {Name: "gneqq", CodePoints: []int{8809}, Characters: []byte{0xe2, 0x89, 0xa9}}, + "gnsim": {Name: "gnsim", CodePoints: []int{8935}, Characters: []byte{0xe2, 0x8b, 0xa7}}, + "gopf": {Name: "gopf", CodePoints: []int{120152}, Characters: []byte{0xf0, 0x9d, 0x95, 0x98}}, + "grave": {Name: "grave", CodePoints: []int{96}, Characters: []byte{0x60}}, + "gscr": {Name: "gscr", CodePoints: []int{8458}, Characters: []byte{0xe2, 0x84, 0x8a}}, + "gsim": {Name: "gsim", CodePoints: []int{8819}, Characters: []byte{0xe2, 0x89, 0xb3}}, + "gsime": {Name: "gsime", CodePoints: []int{10894}, Characters: []byte{0xe2, 0xaa, 0x8e}}, + "gsiml": {Name: "gsiml", CodePoints: []int{10896}, Characters: []byte{0xe2, 0xaa, 0x90}}, + "gt": {Name: "gt", CodePoints: []int{62}, Characters: []byte{0x3e}}, + "gtcc": {Name: "gtcc", CodePoints: []int{10919}, Characters: []byte{0xe2, 0xaa, 0xa7}}, + "gtcir": {Name: "gtcir", CodePoints: []int{10874}, Characters: []byte{0xe2, 0xa9, 0xba}}, + "gtdot": {Name: "gtdot", CodePoints: []int{8919}, Characters: []byte{0xe2, 0x8b, 0x97}}, + "gtlPar": {Name: "gtlPar", CodePoints: []int{10645}, Characters: []byte{0xe2, 0xa6, 0x95}}, + "gtquest": {Name: "gtquest", CodePoints: []int{10876}, Characters: []byte{0xe2, 0xa9, 0xbc}}, + "gtrapprox": {Name: "gtrapprox", CodePoints: []int{10886}, Characters: []byte{0xe2, 0xaa, 0x86}}, + "gtrarr": {Name: "gtrarr", CodePoints: []int{10616}, Characters: []byte{0xe2, 0xa5, 0xb8}}, + "gtrdot": {Name: "gtrdot", CodePoints: []int{8919}, Characters: []byte{0xe2, 0x8b, 0x97}}, + "gtreqless": {Name: "gtreqless", CodePoints: []int{8923}, Characters: []byte{0xe2, 0x8b, 0x9b}}, + "gtreqqless": {Name: "gtreqqless", CodePoints: []int{10892}, Characters: []byte{0xe2, 0xaa, 0x8c}}, + "gtrless": {Name: "gtrless", CodePoints: []int{8823}, Characters: []byte{0xe2, 0x89, 0xb7}}, + "gtrsim": {Name: "gtrsim", CodePoints: []int{8819}, Characters: []byte{0xe2, 0x89, 0xb3}}, + "gvertneqq": {Name: "gvertneqq", CodePoints: []int{8809, 65024}, Characters: []byte{0xe2, 0x89, 0xa9, 0xef, 0xb8, 0x80}}, + "gvnE": {Name: "gvnE", CodePoints: []int{8809, 65024}, Characters: []byte{0xe2, 0x89, 0xa9, 0xef, 0xb8, 0x80}}, + "hArr": {Name: "hArr", CodePoints: []int{8660}, Characters: []byte{0xe2, 0x87, 0x94}}, + "hairsp": {Name: "hairsp", CodePoints: []int{8202}, Characters: []byte{0xe2, 0x80, 0x8a}}, + "half": {Name: "half", CodePoints: []int{189}, Characters: []byte{0xc2, 0xbd}}, + "hamilt": {Name: "hamilt", CodePoints: []int{8459}, Characters: []byte{0xe2, 0x84, 0x8b}}, + "hardcy": {Name: "hardcy", CodePoints: []int{1098}, Characters: []byte{0xd1, 0x8a}}, + "harr": {Name: "harr", CodePoints: []int{8596}, Characters: []byte{0xe2, 0x86, 0x94}}, + "harrcir": {Name: "harrcir", CodePoints: []int{10568}, Characters: []byte{0xe2, 0xa5, 0x88}}, + "harrw": {Name: "harrw", CodePoints: []int{8621}, Characters: []byte{0xe2, 0x86, 0xad}}, + "hbar": {Name: "hbar", CodePoints: []int{8463}, Characters: []byte{0xe2, 0x84, 0x8f}}, + "hcirc": {Name: "hcirc", CodePoints: []int{293}, Characters: []byte{0xc4, 0xa5}}, + "hearts": {Name: "hearts", CodePoints: []int{9829}, Characters: []byte{0xe2, 0x99, 0xa5}}, + "heartsuit": {Name: "heartsuit", CodePoints: []int{9829}, Characters: []byte{0xe2, 0x99, 0xa5}}, + "hellip": {Name: "hellip", CodePoints: []int{8230}, Characters: []byte{0xe2, 0x80, 0xa6}}, + "hercon": {Name: "hercon", CodePoints: []int{8889}, Characters: []byte{0xe2, 0x8a, 0xb9}}, + "hfr": {Name: "hfr", CodePoints: []int{120101}, Characters: []byte{0xf0, 0x9d, 0x94, 0xa5}}, + "hksearow": {Name: "hksearow", CodePoints: []int{10533}, Characters: []byte{0xe2, 0xa4, 0xa5}}, + "hkswarow": {Name: "hkswarow", CodePoints: []int{10534}, Characters: []byte{0xe2, 0xa4, 0xa6}}, + "hoarr": {Name: "hoarr", CodePoints: []int{8703}, Characters: []byte{0xe2, 0x87, 0xbf}}, + "homtht": {Name: "homtht", CodePoints: []int{8763}, Characters: []byte{0xe2, 0x88, 0xbb}}, + "hookleftarrow": {Name: "hookleftarrow", CodePoints: []int{8617}, Characters: []byte{0xe2, 0x86, 0xa9}}, + "hookrightarrow": {Name: "hookrightarrow", CodePoints: []int{8618}, Characters: []byte{0xe2, 0x86, 0xaa}}, + "hopf": {Name: "hopf", CodePoints: []int{120153}, Characters: []byte{0xf0, 0x9d, 0x95, 0x99}}, + "horbar": {Name: "horbar", CodePoints: []int{8213}, Characters: []byte{0xe2, 0x80, 0x95}}, + "hscr": {Name: "hscr", CodePoints: []int{119997}, Characters: []byte{0xf0, 0x9d, 0x92, 0xbd}}, + "hslash": {Name: "hslash", CodePoints: []int{8463}, Characters: []byte{0xe2, 0x84, 0x8f}}, + "hstrok": {Name: "hstrok", CodePoints: []int{295}, Characters: []byte{0xc4, 0xa7}}, + "hybull": {Name: "hybull", CodePoints: []int{8259}, Characters: []byte{0xe2, 0x81, 0x83}}, + "hyphen": {Name: "hyphen", CodePoints: []int{8208}, Characters: []byte{0xe2, 0x80, 0x90}}, + "iacute": {Name: "iacute", CodePoints: []int{237}, Characters: []byte{0xc3, 0xad}}, + "ic": {Name: "ic", CodePoints: []int{8291}, Characters: []byte{0xe2, 0x81, 0xa3}}, + "icirc": {Name: "icirc", CodePoints: []int{238}, Characters: []byte{0xc3, 0xae}}, + "icy": {Name: "icy", CodePoints: []int{1080}, Characters: []byte{0xd0, 0xb8}}, + "iecy": {Name: "iecy", CodePoints: []int{1077}, Characters: []byte{0xd0, 0xb5}}, + "iexcl": {Name: "iexcl", CodePoints: []int{161}, Characters: []byte{0xc2, 0xa1}}, + "iff": {Name: "iff", CodePoints: []int{8660}, Characters: []byte{0xe2, 0x87, 0x94}}, + "ifr": {Name: "ifr", CodePoints: []int{120102}, Characters: []byte{0xf0, 0x9d, 0x94, 0xa6}}, + "igrave": {Name: "igrave", CodePoints: []int{236}, Characters: []byte{0xc3, 0xac}}, + "ii": {Name: "ii", CodePoints: []int{8520}, Characters: []byte{0xe2, 0x85, 0x88}}, + "iiiint": {Name: "iiiint", CodePoints: []int{10764}, Characters: []byte{0xe2, 0xa8, 0x8c}}, + "iiint": {Name: "iiint", CodePoints: []int{8749}, Characters: []byte{0xe2, 0x88, 0xad}}, + "iinfin": {Name: "iinfin", CodePoints: []int{10716}, Characters: []byte{0xe2, 0xa7, 0x9c}}, + "iiota": {Name: "iiota", CodePoints: []int{8489}, Characters: []byte{0xe2, 0x84, 0xa9}}, + "ijlig": {Name: "ijlig", CodePoints: []int{307}, Characters: []byte{0xc4, 0xb3}}, + "imacr": {Name: "imacr", CodePoints: []int{299}, Characters: []byte{0xc4, 0xab}}, + "image": {Name: "image", CodePoints: []int{8465}, Characters: []byte{0xe2, 0x84, 0x91}}, + "imagline": {Name: "imagline", CodePoints: []int{8464}, Characters: []byte{0xe2, 0x84, 0x90}}, + "imagpart": {Name: "imagpart", CodePoints: []int{8465}, Characters: []byte{0xe2, 0x84, 0x91}}, + "imath": {Name: "imath", CodePoints: []int{305}, Characters: []byte{0xc4, 0xb1}}, + "imof": {Name: "imof", CodePoints: []int{8887}, Characters: []byte{0xe2, 0x8a, 0xb7}}, + "imped": {Name: "imped", CodePoints: []int{437}, Characters: []byte{0xc6, 0xb5}}, + "in": {Name: "in", CodePoints: []int{8712}, Characters: []byte{0xe2, 0x88, 0x88}}, + "incare": {Name: "incare", CodePoints: []int{8453}, Characters: []byte{0xe2, 0x84, 0x85}}, + "infin": {Name: "infin", CodePoints: []int{8734}, Characters: []byte{0xe2, 0x88, 0x9e}}, + "infintie": {Name: "infintie", CodePoints: []int{10717}, Characters: []byte{0xe2, 0xa7, 0x9d}}, + "inodot": {Name: "inodot", CodePoints: []int{305}, Characters: []byte{0xc4, 0xb1}}, + "int": {Name: "int", CodePoints: []int{8747}, Characters: []byte{0xe2, 0x88, 0xab}}, + "intcal": {Name: "intcal", CodePoints: []int{8890}, Characters: []byte{0xe2, 0x8a, 0xba}}, + "integers": {Name: "integers", CodePoints: []int{8484}, Characters: []byte{0xe2, 0x84, 0xa4}}, + "intercal": {Name: "intercal", CodePoints: []int{8890}, Characters: []byte{0xe2, 0x8a, 0xba}}, + "intlarhk": {Name: "intlarhk", CodePoints: []int{10775}, Characters: []byte{0xe2, 0xa8, 0x97}}, + "intprod": {Name: "intprod", CodePoints: []int{10812}, Characters: []byte{0xe2, 0xa8, 0xbc}}, + "iocy": {Name: "iocy", CodePoints: []int{1105}, Characters: []byte{0xd1, 0x91}}, + "iogon": {Name: "iogon", CodePoints: []int{303}, Characters: []byte{0xc4, 0xaf}}, + "iopf": {Name: "iopf", CodePoints: []int{120154}, Characters: []byte{0xf0, 0x9d, 0x95, 0x9a}}, + "iota": {Name: "iota", CodePoints: []int{953}, Characters: []byte{0xce, 0xb9}}, + "iprod": {Name: "iprod", CodePoints: []int{10812}, Characters: []byte{0xe2, 0xa8, 0xbc}}, + "iquest": {Name: "iquest", CodePoints: []int{191}, Characters: []byte{0xc2, 0xbf}}, + "iscr": {Name: "iscr", CodePoints: []int{119998}, Characters: []byte{0xf0, 0x9d, 0x92, 0xbe}}, + "isin": {Name: "isin", CodePoints: []int{8712}, Characters: []byte{0xe2, 0x88, 0x88}}, + "isinE": {Name: "isinE", CodePoints: []int{8953}, Characters: []byte{0xe2, 0x8b, 0xb9}}, + "isindot": {Name: "isindot", CodePoints: []int{8949}, Characters: []byte{0xe2, 0x8b, 0xb5}}, + "isins": {Name: "isins", CodePoints: []int{8948}, Characters: []byte{0xe2, 0x8b, 0xb4}}, + "isinsv": {Name: "isinsv", CodePoints: []int{8947}, Characters: []byte{0xe2, 0x8b, 0xb3}}, + "isinv": {Name: "isinv", CodePoints: []int{8712}, Characters: []byte{0xe2, 0x88, 0x88}}, + "it": {Name: "it", CodePoints: []int{8290}, Characters: []byte{0xe2, 0x81, 0xa2}}, + "itilde": {Name: "itilde", CodePoints: []int{297}, Characters: []byte{0xc4, 0xa9}}, + "iukcy": {Name: "iukcy", CodePoints: []int{1110}, Characters: []byte{0xd1, 0x96}}, + "iuml": {Name: "iuml", CodePoints: []int{239}, Characters: []byte{0xc3, 0xaf}}, + "jcirc": {Name: "jcirc", CodePoints: []int{309}, Characters: []byte{0xc4, 0xb5}}, + "jcy": {Name: "jcy", CodePoints: []int{1081}, Characters: []byte{0xd0, 0xb9}}, + "jfr": {Name: "jfr", CodePoints: []int{120103}, Characters: []byte{0xf0, 0x9d, 0x94, 0xa7}}, + "jmath": {Name: "jmath", CodePoints: []int{567}, Characters: []byte{0xc8, 0xb7}}, + "jopf": {Name: "jopf", CodePoints: []int{120155}, Characters: []byte{0xf0, 0x9d, 0x95, 0x9b}}, + "jscr": {Name: "jscr", CodePoints: []int{119999}, Characters: []byte{0xf0, 0x9d, 0x92, 0xbf}}, + "jsercy": {Name: "jsercy", CodePoints: []int{1112}, Characters: []byte{0xd1, 0x98}}, + "jukcy": {Name: "jukcy", CodePoints: []int{1108}, Characters: []byte{0xd1, 0x94}}, + "kappa": {Name: "kappa", CodePoints: []int{954}, Characters: []byte{0xce, 0xba}}, + "kappav": {Name: "kappav", CodePoints: []int{1008}, Characters: []byte{0xcf, 0xb0}}, + "kcedil": {Name: "kcedil", CodePoints: []int{311}, Characters: []byte{0xc4, 0xb7}}, + "kcy": {Name: "kcy", CodePoints: []int{1082}, Characters: []byte{0xd0, 0xba}}, + "kfr": {Name: "kfr", CodePoints: []int{120104}, Characters: []byte{0xf0, 0x9d, 0x94, 0xa8}}, + "kgreen": {Name: "kgreen", CodePoints: []int{312}, Characters: []byte{0xc4, 0xb8}}, + "khcy": {Name: "khcy", CodePoints: []int{1093}, Characters: []byte{0xd1, 0x85}}, + "kjcy": {Name: "kjcy", CodePoints: []int{1116}, Characters: []byte{0xd1, 0x9c}}, + "kopf": {Name: "kopf", CodePoints: []int{120156}, Characters: []byte{0xf0, 0x9d, 0x95, 0x9c}}, + "kscr": {Name: "kscr", CodePoints: []int{120000}, Characters: []byte{0xf0, 0x9d, 0x93, 0x80}}, + "lAarr": {Name: "lAarr", CodePoints: []int{8666}, Characters: []byte{0xe2, 0x87, 0x9a}}, + "lArr": {Name: "lArr", CodePoints: []int{8656}, Characters: []byte{0xe2, 0x87, 0x90}}, + "lAtail": {Name: "lAtail", CodePoints: []int{10523}, Characters: []byte{0xe2, 0xa4, 0x9b}}, + "lBarr": {Name: "lBarr", CodePoints: []int{10510}, Characters: []byte{0xe2, 0xa4, 0x8e}}, + "lE": {Name: "lE", CodePoints: []int{8806}, Characters: []byte{0xe2, 0x89, 0xa6}}, + "lEg": {Name: "lEg", CodePoints: []int{10891}, Characters: []byte{0xe2, 0xaa, 0x8b}}, + "lHar": {Name: "lHar", CodePoints: []int{10594}, Characters: []byte{0xe2, 0xa5, 0xa2}}, + "lacute": {Name: "lacute", CodePoints: []int{314}, Characters: []byte{0xc4, 0xba}}, + "laemptyv": {Name: "laemptyv", CodePoints: []int{10676}, Characters: []byte{0xe2, 0xa6, 0xb4}}, + "lagran": {Name: "lagran", CodePoints: []int{8466}, Characters: []byte{0xe2, 0x84, 0x92}}, + "lambda": {Name: "lambda", CodePoints: []int{955}, Characters: []byte{0xce, 0xbb}}, + "lang": {Name: "lang", CodePoints: []int{10216}, Characters: []byte{0xe2, 0x9f, 0xa8}}, + "langd": {Name: "langd", CodePoints: []int{10641}, Characters: []byte{0xe2, 0xa6, 0x91}}, + "langle": {Name: "langle", CodePoints: []int{10216}, Characters: []byte{0xe2, 0x9f, 0xa8}}, + "lap": {Name: "lap", CodePoints: []int{10885}, Characters: []byte{0xe2, 0xaa, 0x85}}, + "laquo": {Name: "laquo", CodePoints: []int{171}, Characters: []byte{0xc2, 0xab}}, + "larr": {Name: "larr", CodePoints: []int{8592}, Characters: []byte{0xe2, 0x86, 0x90}}, + "larrb": {Name: "larrb", CodePoints: []int{8676}, Characters: []byte{0xe2, 0x87, 0xa4}}, + "larrbfs": {Name: "larrbfs", CodePoints: []int{10527}, Characters: []byte{0xe2, 0xa4, 0x9f}}, + "larrfs": {Name: "larrfs", CodePoints: []int{10525}, Characters: []byte{0xe2, 0xa4, 0x9d}}, + "larrhk": {Name: "larrhk", CodePoints: []int{8617}, Characters: []byte{0xe2, 0x86, 0xa9}}, + "larrlp": {Name: "larrlp", CodePoints: []int{8619}, Characters: []byte{0xe2, 0x86, 0xab}}, + "larrpl": {Name: "larrpl", CodePoints: []int{10553}, Characters: []byte{0xe2, 0xa4, 0xb9}}, + "larrsim": {Name: "larrsim", CodePoints: []int{10611}, Characters: []byte{0xe2, 0xa5, 0xb3}}, + "larrtl": {Name: "larrtl", CodePoints: []int{8610}, Characters: []byte{0xe2, 0x86, 0xa2}}, + "lat": {Name: "lat", CodePoints: []int{10923}, Characters: []byte{0xe2, 0xaa, 0xab}}, + "latail": {Name: "latail", CodePoints: []int{10521}, Characters: []byte{0xe2, 0xa4, 0x99}}, + "late": {Name: "late", CodePoints: []int{10925}, Characters: []byte{0xe2, 0xaa, 0xad}}, + "lates": {Name: "lates", CodePoints: []int{10925, 65024}, Characters: []byte{0xe2, 0xaa, 0xad, 0xef, 0xb8, 0x80}}, + "lbarr": {Name: "lbarr", CodePoints: []int{10508}, Characters: []byte{0xe2, 0xa4, 0x8c}}, + "lbbrk": {Name: "lbbrk", CodePoints: []int{10098}, Characters: []byte{0xe2, 0x9d, 0xb2}}, + "lbrace": {Name: "lbrace", CodePoints: []int{123}, Characters: []byte{0x7b}}, + "lbrack": {Name: "lbrack", CodePoints: []int{91}, Characters: []byte{0x5b}}, + "lbrke": {Name: "lbrke", CodePoints: []int{10635}, Characters: []byte{0xe2, 0xa6, 0x8b}}, + "lbrksld": {Name: "lbrksld", CodePoints: []int{10639}, Characters: []byte{0xe2, 0xa6, 0x8f}}, + "lbrkslu": {Name: "lbrkslu", CodePoints: []int{10637}, Characters: []byte{0xe2, 0xa6, 0x8d}}, + "lcaron": {Name: "lcaron", CodePoints: []int{318}, Characters: []byte{0xc4, 0xbe}}, + "lcedil": {Name: "lcedil", CodePoints: []int{316}, Characters: []byte{0xc4, 0xbc}}, + "lceil": {Name: "lceil", CodePoints: []int{8968}, Characters: []byte{0xe2, 0x8c, 0x88}}, + "lcub": {Name: "lcub", CodePoints: []int{123}, Characters: []byte{0x7b}}, + "lcy": {Name: "lcy", CodePoints: []int{1083}, Characters: []byte{0xd0, 0xbb}}, + "ldca": {Name: "ldca", CodePoints: []int{10550}, Characters: []byte{0xe2, 0xa4, 0xb6}}, + "ldquo": {Name: "ldquo", CodePoints: []int{8220}, Characters: []byte{0xe2, 0x80, 0x9c}}, + "ldquor": {Name: "ldquor", CodePoints: []int{8222}, Characters: []byte{0xe2, 0x80, 0x9e}}, + "ldrdhar": {Name: "ldrdhar", CodePoints: []int{10599}, Characters: []byte{0xe2, 0xa5, 0xa7}}, + "ldrushar": {Name: "ldrushar", CodePoints: []int{10571}, Characters: []byte{0xe2, 0xa5, 0x8b}}, + "ldsh": {Name: "ldsh", CodePoints: []int{8626}, Characters: []byte{0xe2, 0x86, 0xb2}}, + "le": {Name: "le", CodePoints: []int{8804}, Characters: []byte{0xe2, 0x89, 0xa4}}, + "leftarrow": {Name: "leftarrow", CodePoints: []int{8592}, Characters: []byte{0xe2, 0x86, 0x90}}, + "leftarrowtail": {Name: "leftarrowtail", CodePoints: []int{8610}, Characters: []byte{0xe2, 0x86, 0xa2}}, + "leftharpoondown": {Name: "leftharpoondown", CodePoints: []int{8637}, Characters: []byte{0xe2, 0x86, 0xbd}}, + "leftharpoonup": {Name: "leftharpoonup", CodePoints: []int{8636}, Characters: []byte{0xe2, 0x86, 0xbc}}, + "leftleftarrows": {Name: "leftleftarrows", CodePoints: []int{8647}, Characters: []byte{0xe2, 0x87, 0x87}}, + "leftrightarrow": {Name: "leftrightarrow", CodePoints: []int{8596}, Characters: []byte{0xe2, 0x86, 0x94}}, + "leftrightarrows": {Name: "leftrightarrows", CodePoints: []int{8646}, Characters: []byte{0xe2, 0x87, 0x86}}, + "leftrightharpoons": {Name: "leftrightharpoons", CodePoints: []int{8651}, Characters: []byte{0xe2, 0x87, 0x8b}}, + "leftrightsquigarrow": {Name: "leftrightsquigarrow", CodePoints: []int{8621}, Characters: []byte{0xe2, 0x86, 0xad}}, + "leftthreetimes": {Name: "leftthreetimes", CodePoints: []int{8907}, Characters: []byte{0xe2, 0x8b, 0x8b}}, + "leg": {Name: "leg", CodePoints: []int{8922}, Characters: []byte{0xe2, 0x8b, 0x9a}}, + "leq": {Name: "leq", CodePoints: []int{8804}, Characters: []byte{0xe2, 0x89, 0xa4}}, + "leqq": {Name: "leqq", CodePoints: []int{8806}, Characters: []byte{0xe2, 0x89, 0xa6}}, + "leqslant": {Name: "leqslant", CodePoints: []int{10877}, Characters: []byte{0xe2, 0xa9, 0xbd}}, + "les": {Name: "les", CodePoints: []int{10877}, Characters: []byte{0xe2, 0xa9, 0xbd}}, + "lescc": {Name: "lescc", CodePoints: []int{10920}, Characters: []byte{0xe2, 0xaa, 0xa8}}, + "lesdot": {Name: "lesdot", CodePoints: []int{10879}, Characters: []byte{0xe2, 0xa9, 0xbf}}, + "lesdoto": {Name: "lesdoto", CodePoints: []int{10881}, Characters: []byte{0xe2, 0xaa, 0x81}}, + "lesdotor": {Name: "lesdotor", CodePoints: []int{10883}, Characters: []byte{0xe2, 0xaa, 0x83}}, + "lesg": {Name: "lesg", CodePoints: []int{8922, 65024}, Characters: []byte{0xe2, 0x8b, 0x9a, 0xef, 0xb8, 0x80}}, + "lesges": {Name: "lesges", CodePoints: []int{10899}, Characters: []byte{0xe2, 0xaa, 0x93}}, + "lessapprox": {Name: "lessapprox", CodePoints: []int{10885}, Characters: []byte{0xe2, 0xaa, 0x85}}, + "lessdot": {Name: "lessdot", CodePoints: []int{8918}, Characters: []byte{0xe2, 0x8b, 0x96}}, + "lesseqgtr": {Name: "lesseqgtr", CodePoints: []int{8922}, Characters: []byte{0xe2, 0x8b, 0x9a}}, + "lesseqqgtr": {Name: "lesseqqgtr", CodePoints: []int{10891}, Characters: []byte{0xe2, 0xaa, 0x8b}}, + "lessgtr": {Name: "lessgtr", CodePoints: []int{8822}, Characters: []byte{0xe2, 0x89, 0xb6}}, + "lesssim": {Name: "lesssim", CodePoints: []int{8818}, Characters: []byte{0xe2, 0x89, 0xb2}}, + "lfisht": {Name: "lfisht", CodePoints: []int{10620}, Characters: []byte{0xe2, 0xa5, 0xbc}}, + "lfloor": {Name: "lfloor", CodePoints: []int{8970}, Characters: []byte{0xe2, 0x8c, 0x8a}}, + "lfr": {Name: "lfr", CodePoints: []int{120105}, Characters: []byte{0xf0, 0x9d, 0x94, 0xa9}}, + "lg": {Name: "lg", CodePoints: []int{8822}, Characters: []byte{0xe2, 0x89, 0xb6}}, + "lgE": {Name: "lgE", CodePoints: []int{10897}, Characters: []byte{0xe2, 0xaa, 0x91}}, + "lhard": {Name: "lhard", CodePoints: []int{8637}, Characters: []byte{0xe2, 0x86, 0xbd}}, + "lharu": {Name: "lharu", CodePoints: []int{8636}, Characters: []byte{0xe2, 0x86, 0xbc}}, + "lharul": {Name: "lharul", CodePoints: []int{10602}, Characters: []byte{0xe2, 0xa5, 0xaa}}, + "lhblk": {Name: "lhblk", CodePoints: []int{9604}, Characters: []byte{0xe2, 0x96, 0x84}}, + "ljcy": {Name: "ljcy", CodePoints: []int{1113}, Characters: []byte{0xd1, 0x99}}, + "ll": {Name: "ll", CodePoints: []int{8810}, Characters: []byte{0xe2, 0x89, 0xaa}}, + "llarr": {Name: "llarr", CodePoints: []int{8647}, Characters: []byte{0xe2, 0x87, 0x87}}, + "llcorner": {Name: "llcorner", CodePoints: []int{8990}, Characters: []byte{0xe2, 0x8c, 0x9e}}, + "llhard": {Name: "llhard", CodePoints: []int{10603}, Characters: []byte{0xe2, 0xa5, 0xab}}, + "lltri": {Name: "lltri", CodePoints: []int{9722}, Characters: []byte{0xe2, 0x97, 0xba}}, + "lmidot": {Name: "lmidot", CodePoints: []int{320}, Characters: []byte{0xc5, 0x80}}, + "lmoust": {Name: "lmoust", CodePoints: []int{9136}, Characters: []byte{0xe2, 0x8e, 0xb0}}, + "lmoustache": {Name: "lmoustache", CodePoints: []int{9136}, Characters: []byte{0xe2, 0x8e, 0xb0}}, + "lnE": {Name: "lnE", CodePoints: []int{8808}, Characters: []byte{0xe2, 0x89, 0xa8}}, + "lnap": {Name: "lnap", CodePoints: []int{10889}, Characters: []byte{0xe2, 0xaa, 0x89}}, + "lnapprox": {Name: "lnapprox", CodePoints: []int{10889}, Characters: []byte{0xe2, 0xaa, 0x89}}, + "lne": {Name: "lne", CodePoints: []int{10887}, Characters: []byte{0xe2, 0xaa, 0x87}}, + "lneq": {Name: "lneq", CodePoints: []int{10887}, Characters: []byte{0xe2, 0xaa, 0x87}}, + "lneqq": {Name: "lneqq", CodePoints: []int{8808}, Characters: []byte{0xe2, 0x89, 0xa8}}, + "lnsim": {Name: "lnsim", CodePoints: []int{8934}, Characters: []byte{0xe2, 0x8b, 0xa6}}, + "loang": {Name: "loang", CodePoints: []int{10220}, Characters: []byte{0xe2, 0x9f, 0xac}}, + "loarr": {Name: "loarr", CodePoints: []int{8701}, Characters: []byte{0xe2, 0x87, 0xbd}}, + "lobrk": {Name: "lobrk", CodePoints: []int{10214}, Characters: []byte{0xe2, 0x9f, 0xa6}}, + "longleftarrow": {Name: "longleftarrow", CodePoints: []int{10229}, Characters: []byte{0xe2, 0x9f, 0xb5}}, + "longleftrightarrow": {Name: "longleftrightarrow", CodePoints: []int{10231}, Characters: []byte{0xe2, 0x9f, 0xb7}}, + "longmapsto": {Name: "longmapsto", CodePoints: []int{10236}, Characters: []byte{0xe2, 0x9f, 0xbc}}, + "longrightarrow": {Name: "longrightarrow", CodePoints: []int{10230}, Characters: []byte{0xe2, 0x9f, 0xb6}}, + "looparrowleft": {Name: "looparrowleft", CodePoints: []int{8619}, Characters: []byte{0xe2, 0x86, 0xab}}, + "looparrowright": {Name: "looparrowright", CodePoints: []int{8620}, Characters: []byte{0xe2, 0x86, 0xac}}, + "lopar": {Name: "lopar", CodePoints: []int{10629}, Characters: []byte{0xe2, 0xa6, 0x85}}, + "lopf": {Name: "lopf", CodePoints: []int{120157}, Characters: []byte{0xf0, 0x9d, 0x95, 0x9d}}, + "loplus": {Name: "loplus", CodePoints: []int{10797}, Characters: []byte{0xe2, 0xa8, 0xad}}, + "lotimes": {Name: "lotimes", CodePoints: []int{10804}, Characters: []byte{0xe2, 0xa8, 0xb4}}, + "lowast": {Name: "lowast", CodePoints: []int{8727}, Characters: []byte{0xe2, 0x88, 0x97}}, + "lowbar": {Name: "lowbar", CodePoints: []int{95}, Characters: []byte{0x5f}}, + "loz": {Name: "loz", CodePoints: []int{9674}, Characters: []byte{0xe2, 0x97, 0x8a}}, + "lozenge": {Name: "lozenge", CodePoints: []int{9674}, Characters: []byte{0xe2, 0x97, 0x8a}}, + "lozf": {Name: "lozf", CodePoints: []int{10731}, Characters: []byte{0xe2, 0xa7, 0xab}}, + "lpar": {Name: "lpar", CodePoints: []int{40}, Characters: []byte{0x28}}, + "lparlt": {Name: "lparlt", CodePoints: []int{10643}, Characters: []byte{0xe2, 0xa6, 0x93}}, + "lrarr": {Name: "lrarr", CodePoints: []int{8646}, Characters: []byte{0xe2, 0x87, 0x86}}, + "lrcorner": {Name: "lrcorner", CodePoints: []int{8991}, Characters: []byte{0xe2, 0x8c, 0x9f}}, + "lrhar": {Name: "lrhar", CodePoints: []int{8651}, Characters: []byte{0xe2, 0x87, 0x8b}}, + "lrhard": {Name: "lrhard", CodePoints: []int{10605}, Characters: []byte{0xe2, 0xa5, 0xad}}, + "lrm": {Name: "lrm", CodePoints: []int{8206}, Characters: []byte{0xe2, 0x80, 0x8e}}, + "lrtri": {Name: "lrtri", CodePoints: []int{8895}, Characters: []byte{0xe2, 0x8a, 0xbf}}, + "lsaquo": {Name: "lsaquo", CodePoints: []int{8249}, Characters: []byte{0xe2, 0x80, 0xb9}}, + "lscr": {Name: "lscr", CodePoints: []int{120001}, Characters: []byte{0xf0, 0x9d, 0x93, 0x81}}, + "lsh": {Name: "lsh", CodePoints: []int{8624}, Characters: []byte{0xe2, 0x86, 0xb0}}, + "lsim": {Name: "lsim", CodePoints: []int{8818}, Characters: []byte{0xe2, 0x89, 0xb2}}, + "lsime": {Name: "lsime", CodePoints: []int{10893}, Characters: []byte{0xe2, 0xaa, 0x8d}}, + "lsimg": {Name: "lsimg", CodePoints: []int{10895}, Characters: []byte{0xe2, 0xaa, 0x8f}}, + "lsqb": {Name: "lsqb", CodePoints: []int{91}, Characters: []byte{0x5b}}, + "lsquo": {Name: "lsquo", CodePoints: []int{8216}, Characters: []byte{0xe2, 0x80, 0x98}}, + "lsquor": {Name: "lsquor", CodePoints: []int{8218}, Characters: []byte{0xe2, 0x80, 0x9a}}, + "lstrok": {Name: "lstrok", CodePoints: []int{322}, Characters: []byte{0xc5, 0x82}}, + "lt": {Name: "lt", CodePoints: []int{60}, Characters: []byte{0x3c}}, + "ltcc": {Name: "ltcc", CodePoints: []int{10918}, Characters: []byte{0xe2, 0xaa, 0xa6}}, + "ltcir": {Name: "ltcir", CodePoints: []int{10873}, Characters: []byte{0xe2, 0xa9, 0xb9}}, + "ltdot": {Name: "ltdot", CodePoints: []int{8918}, Characters: []byte{0xe2, 0x8b, 0x96}}, + "lthree": {Name: "lthree", CodePoints: []int{8907}, Characters: []byte{0xe2, 0x8b, 0x8b}}, + "ltimes": {Name: "ltimes", CodePoints: []int{8905}, Characters: []byte{0xe2, 0x8b, 0x89}}, + "ltlarr": {Name: "ltlarr", CodePoints: []int{10614}, Characters: []byte{0xe2, 0xa5, 0xb6}}, + "ltquest": {Name: "ltquest", CodePoints: []int{10875}, Characters: []byte{0xe2, 0xa9, 0xbb}}, + "ltrPar": {Name: "ltrPar", CodePoints: []int{10646}, Characters: []byte{0xe2, 0xa6, 0x96}}, + "ltri": {Name: "ltri", CodePoints: []int{9667}, Characters: []byte{0xe2, 0x97, 0x83}}, + "ltrie": {Name: "ltrie", CodePoints: []int{8884}, Characters: []byte{0xe2, 0x8a, 0xb4}}, + "ltrif": {Name: "ltrif", CodePoints: []int{9666}, Characters: []byte{0xe2, 0x97, 0x82}}, + "lurdshar": {Name: "lurdshar", CodePoints: []int{10570}, Characters: []byte{0xe2, 0xa5, 0x8a}}, + "luruhar": {Name: "luruhar", CodePoints: []int{10598}, Characters: []byte{0xe2, 0xa5, 0xa6}}, + "lvertneqq": {Name: "lvertneqq", CodePoints: []int{8808, 65024}, Characters: []byte{0xe2, 0x89, 0xa8, 0xef, 0xb8, 0x80}}, + "lvnE": {Name: "lvnE", CodePoints: []int{8808, 65024}, Characters: []byte{0xe2, 0x89, 0xa8, 0xef, 0xb8, 0x80}}, + "mDDot": {Name: "mDDot", CodePoints: []int{8762}, Characters: []byte{0xe2, 0x88, 0xba}}, + "macr": {Name: "macr", CodePoints: []int{175}, Characters: []byte{0xc2, 0xaf}}, + "male": {Name: "male", CodePoints: []int{9794}, Characters: []byte{0xe2, 0x99, 0x82}}, + "malt": {Name: "malt", CodePoints: []int{10016}, Characters: []byte{0xe2, 0x9c, 0xa0}}, + "maltese": {Name: "maltese", CodePoints: []int{10016}, Characters: []byte{0xe2, 0x9c, 0xa0}}, + "map": {Name: "map", CodePoints: []int{8614}, Characters: []byte{0xe2, 0x86, 0xa6}}, + "mapsto": {Name: "mapsto", CodePoints: []int{8614}, Characters: []byte{0xe2, 0x86, 0xa6}}, + "mapstodown": {Name: "mapstodown", CodePoints: []int{8615}, Characters: []byte{0xe2, 0x86, 0xa7}}, + "mapstoleft": {Name: "mapstoleft", CodePoints: []int{8612}, Characters: []byte{0xe2, 0x86, 0xa4}}, + "mapstoup": {Name: "mapstoup", CodePoints: []int{8613}, Characters: []byte{0xe2, 0x86, 0xa5}}, + "marker": {Name: "marker", CodePoints: []int{9646}, Characters: []byte{0xe2, 0x96, 0xae}}, + "mcomma": {Name: "mcomma", CodePoints: []int{10793}, Characters: []byte{0xe2, 0xa8, 0xa9}}, + "mcy": {Name: "mcy", CodePoints: []int{1084}, Characters: []byte{0xd0, 0xbc}}, + "mdash": {Name: "mdash", CodePoints: []int{8212}, Characters: []byte{0xe2, 0x80, 0x94}}, + "measuredangle": {Name: "measuredangle", CodePoints: []int{8737}, Characters: []byte{0xe2, 0x88, 0xa1}}, + "mfr": {Name: "mfr", CodePoints: []int{120106}, Characters: []byte{0xf0, 0x9d, 0x94, 0xaa}}, + "mho": {Name: "mho", CodePoints: []int{8487}, Characters: []byte{0xe2, 0x84, 0xa7}}, + "micro": {Name: "micro", CodePoints: []int{181}, Characters: []byte{0xc2, 0xb5}}, + "mid": {Name: "mid", CodePoints: []int{8739}, Characters: []byte{0xe2, 0x88, 0xa3}}, + "midast": {Name: "midast", CodePoints: []int{42}, Characters: []byte{0x2a}}, + "midcir": {Name: "midcir", CodePoints: []int{10992}, Characters: []byte{0xe2, 0xab, 0xb0}}, + "middot": {Name: "middot", CodePoints: []int{183}, Characters: []byte{0xc2, 0xb7}}, + "minus": {Name: "minus", CodePoints: []int{8722}, Characters: []byte{0xe2, 0x88, 0x92}}, + "minusb": {Name: "minusb", CodePoints: []int{8863}, Characters: []byte{0xe2, 0x8a, 0x9f}}, + "minusd": {Name: "minusd", CodePoints: []int{8760}, Characters: []byte{0xe2, 0x88, 0xb8}}, + "minusdu": {Name: "minusdu", CodePoints: []int{10794}, Characters: []byte{0xe2, 0xa8, 0xaa}}, + "mlcp": {Name: "mlcp", CodePoints: []int{10971}, Characters: []byte{0xe2, 0xab, 0x9b}}, + "mldr": {Name: "mldr", CodePoints: []int{8230}, Characters: []byte{0xe2, 0x80, 0xa6}}, + "mnplus": {Name: "mnplus", CodePoints: []int{8723}, Characters: []byte{0xe2, 0x88, 0x93}}, + "models": {Name: "models", CodePoints: []int{8871}, Characters: []byte{0xe2, 0x8a, 0xa7}}, + "mopf": {Name: "mopf", CodePoints: []int{120158}, Characters: []byte{0xf0, 0x9d, 0x95, 0x9e}}, + "mp": {Name: "mp", CodePoints: []int{8723}, Characters: []byte{0xe2, 0x88, 0x93}}, + "mscr": {Name: "mscr", CodePoints: []int{120002}, Characters: []byte{0xf0, 0x9d, 0x93, 0x82}}, + "mstpos": {Name: "mstpos", CodePoints: []int{8766}, Characters: []byte{0xe2, 0x88, 0xbe}}, + "mu": {Name: "mu", CodePoints: []int{956}, Characters: []byte{0xce, 0xbc}}, + "multimap": {Name: "multimap", CodePoints: []int{8888}, Characters: []byte{0xe2, 0x8a, 0xb8}}, + "mumap": {Name: "mumap", CodePoints: []int{8888}, Characters: []byte{0xe2, 0x8a, 0xb8}}, + "nGg": {Name: "nGg", CodePoints: []int{8921, 824}, Characters: []byte{0xe2, 0x8b, 0x99, 0xcc, 0xb8}}, + "nGt": {Name: "nGt", CodePoints: []int{8811, 8402}, Characters: []byte{0xe2, 0x89, 0xab, 0xe2, 0x83, 0x92}}, + "nGtv": {Name: "nGtv", CodePoints: []int{8811, 824}, Characters: []byte{0xe2, 0x89, 0xab, 0xcc, 0xb8}}, + "nLeftarrow": {Name: "nLeftarrow", CodePoints: []int{8653}, Characters: []byte{0xe2, 0x87, 0x8d}}, + "nLeftrightarrow": {Name: "nLeftrightarrow", CodePoints: []int{8654}, Characters: []byte{0xe2, 0x87, 0x8e}}, + "nLl": {Name: "nLl", CodePoints: []int{8920, 824}, Characters: []byte{0xe2, 0x8b, 0x98, 0xcc, 0xb8}}, + "nLt": {Name: "nLt", CodePoints: []int{8810, 8402}, Characters: []byte{0xe2, 0x89, 0xaa, 0xe2, 0x83, 0x92}}, + "nLtv": {Name: "nLtv", CodePoints: []int{8810, 824}, Characters: []byte{0xe2, 0x89, 0xaa, 0xcc, 0xb8}}, + "nRightarrow": {Name: "nRightarrow", CodePoints: []int{8655}, Characters: []byte{0xe2, 0x87, 0x8f}}, + "nVDash": {Name: "nVDash", CodePoints: []int{8879}, Characters: []byte{0xe2, 0x8a, 0xaf}}, + "nVdash": {Name: "nVdash", CodePoints: []int{8878}, Characters: []byte{0xe2, 0x8a, 0xae}}, + "nabla": {Name: "nabla", CodePoints: []int{8711}, Characters: []byte{0xe2, 0x88, 0x87}}, + "nacute": {Name: "nacute", CodePoints: []int{324}, Characters: []byte{0xc5, 0x84}}, + "nang": {Name: "nang", CodePoints: []int{8736, 8402}, Characters: []byte{0xe2, 0x88, 0xa0, 0xe2, 0x83, 0x92}}, + "nap": {Name: "nap", CodePoints: []int{8777}, Characters: []byte{0xe2, 0x89, 0x89}}, + "napE": {Name: "napE", CodePoints: []int{10864, 824}, Characters: []byte{0xe2, 0xa9, 0xb0, 0xcc, 0xb8}}, + "napid": {Name: "napid", CodePoints: []int{8779, 824}, Characters: []byte{0xe2, 0x89, 0x8b, 0xcc, 0xb8}}, + "napos": {Name: "napos", CodePoints: []int{329}, Characters: []byte{0xc5, 0x89}}, + "napprox": {Name: "napprox", CodePoints: []int{8777}, Characters: []byte{0xe2, 0x89, 0x89}}, + "natur": {Name: "natur", CodePoints: []int{9838}, Characters: []byte{0xe2, 0x99, 0xae}}, + "natural": {Name: "natural", CodePoints: []int{9838}, Characters: []byte{0xe2, 0x99, 0xae}}, + "naturals": {Name: "naturals", CodePoints: []int{8469}, Characters: []byte{0xe2, 0x84, 0x95}}, + "nbsp": {Name: "nbsp", CodePoints: []int{160}, Characters: []byte{0xc2, 0xa0}}, + "nbump": {Name: "nbump", CodePoints: []int{8782, 824}, Characters: []byte{0xe2, 0x89, 0x8e, 0xcc, 0xb8}}, + "nbumpe": {Name: "nbumpe", CodePoints: []int{8783, 824}, Characters: []byte{0xe2, 0x89, 0x8f, 0xcc, 0xb8}}, + "ncap": {Name: "ncap", CodePoints: []int{10819}, Characters: []byte{0xe2, 0xa9, 0x83}}, + "ncaron": {Name: "ncaron", CodePoints: []int{328}, Characters: []byte{0xc5, 0x88}}, + "ncedil": {Name: "ncedil", CodePoints: []int{326}, Characters: []byte{0xc5, 0x86}}, + "ncong": {Name: "ncong", CodePoints: []int{8775}, Characters: []byte{0xe2, 0x89, 0x87}}, + "ncongdot": {Name: "ncongdot", CodePoints: []int{10861, 824}, Characters: []byte{0xe2, 0xa9, 0xad, 0xcc, 0xb8}}, + "ncup": {Name: "ncup", CodePoints: []int{10818}, Characters: []byte{0xe2, 0xa9, 0x82}}, + "ncy": {Name: "ncy", CodePoints: []int{1085}, Characters: []byte{0xd0, 0xbd}}, + "ndash": {Name: "ndash", CodePoints: []int{8211}, Characters: []byte{0xe2, 0x80, 0x93}}, + "ne": {Name: "ne", CodePoints: []int{8800}, Characters: []byte{0xe2, 0x89, 0xa0}}, + "neArr": {Name: "neArr", CodePoints: []int{8663}, Characters: []byte{0xe2, 0x87, 0x97}}, + "nearhk": {Name: "nearhk", CodePoints: []int{10532}, Characters: []byte{0xe2, 0xa4, 0xa4}}, + "nearr": {Name: "nearr", CodePoints: []int{8599}, Characters: []byte{0xe2, 0x86, 0x97}}, + "nearrow": {Name: "nearrow", CodePoints: []int{8599}, Characters: []byte{0xe2, 0x86, 0x97}}, + "nedot": {Name: "nedot", CodePoints: []int{8784, 824}, Characters: []byte{0xe2, 0x89, 0x90, 0xcc, 0xb8}}, + "nequiv": {Name: "nequiv", CodePoints: []int{8802}, Characters: []byte{0xe2, 0x89, 0xa2}}, + "nesear": {Name: "nesear", CodePoints: []int{10536}, Characters: []byte{0xe2, 0xa4, 0xa8}}, + "nesim": {Name: "nesim", CodePoints: []int{8770, 824}, Characters: []byte{0xe2, 0x89, 0x82, 0xcc, 0xb8}}, + "nexist": {Name: "nexist", CodePoints: []int{8708}, Characters: []byte{0xe2, 0x88, 0x84}}, + "nexists": {Name: "nexists", CodePoints: []int{8708}, Characters: []byte{0xe2, 0x88, 0x84}}, + "nfr": {Name: "nfr", CodePoints: []int{120107}, Characters: []byte{0xf0, 0x9d, 0x94, 0xab}}, + "ngE": {Name: "ngE", CodePoints: []int{8807, 824}, Characters: []byte{0xe2, 0x89, 0xa7, 0xcc, 0xb8}}, + "nge": {Name: "nge", CodePoints: []int{8817}, Characters: []byte{0xe2, 0x89, 0xb1}}, + "ngeq": {Name: "ngeq", CodePoints: []int{8817}, Characters: []byte{0xe2, 0x89, 0xb1}}, + "ngeqq": {Name: "ngeqq", CodePoints: []int{8807, 824}, Characters: []byte{0xe2, 0x89, 0xa7, 0xcc, 0xb8}}, + "ngeqslant": {Name: "ngeqslant", CodePoints: []int{10878, 824}, Characters: []byte{0xe2, 0xa9, 0xbe, 0xcc, 0xb8}}, + "nges": {Name: "nges", CodePoints: []int{10878, 824}, Characters: []byte{0xe2, 0xa9, 0xbe, 0xcc, 0xb8}}, + "ngsim": {Name: "ngsim", CodePoints: []int{8821}, Characters: []byte{0xe2, 0x89, 0xb5}}, + "ngt": {Name: "ngt", CodePoints: []int{8815}, Characters: []byte{0xe2, 0x89, 0xaf}}, + "ngtr": {Name: "ngtr", CodePoints: []int{8815}, Characters: []byte{0xe2, 0x89, 0xaf}}, + "nhArr": {Name: "nhArr", CodePoints: []int{8654}, Characters: []byte{0xe2, 0x87, 0x8e}}, + "nharr": {Name: "nharr", CodePoints: []int{8622}, Characters: []byte{0xe2, 0x86, 0xae}}, + "nhpar": {Name: "nhpar", CodePoints: []int{10994}, Characters: []byte{0xe2, 0xab, 0xb2}}, + "ni": {Name: "ni", CodePoints: []int{8715}, Characters: []byte{0xe2, 0x88, 0x8b}}, + "nis": {Name: "nis", CodePoints: []int{8956}, Characters: []byte{0xe2, 0x8b, 0xbc}}, + "nisd": {Name: "nisd", CodePoints: []int{8954}, Characters: []byte{0xe2, 0x8b, 0xba}}, + "niv": {Name: "niv", CodePoints: []int{8715}, Characters: []byte{0xe2, 0x88, 0x8b}}, + "njcy": {Name: "njcy", CodePoints: []int{1114}, Characters: []byte{0xd1, 0x9a}}, + "nlArr": {Name: "nlArr", CodePoints: []int{8653}, Characters: []byte{0xe2, 0x87, 0x8d}}, + "nlE": {Name: "nlE", CodePoints: []int{8806, 824}, Characters: []byte{0xe2, 0x89, 0xa6, 0xcc, 0xb8}}, + "nlarr": {Name: "nlarr", CodePoints: []int{8602}, Characters: []byte{0xe2, 0x86, 0x9a}}, + "nldr": {Name: "nldr", CodePoints: []int{8229}, Characters: []byte{0xe2, 0x80, 0xa5}}, + "nle": {Name: "nle", CodePoints: []int{8816}, Characters: []byte{0xe2, 0x89, 0xb0}}, + "nleftarrow": {Name: "nleftarrow", CodePoints: []int{8602}, Characters: []byte{0xe2, 0x86, 0x9a}}, + "nleftrightarrow": {Name: "nleftrightarrow", CodePoints: []int{8622}, Characters: []byte{0xe2, 0x86, 0xae}}, + "nleq": {Name: "nleq", CodePoints: []int{8816}, Characters: []byte{0xe2, 0x89, 0xb0}}, + "nleqq": {Name: "nleqq", CodePoints: []int{8806, 824}, Characters: []byte{0xe2, 0x89, 0xa6, 0xcc, 0xb8}}, + "nleqslant": {Name: "nleqslant", CodePoints: []int{10877, 824}, Characters: []byte{0xe2, 0xa9, 0xbd, 0xcc, 0xb8}}, + "nles": {Name: "nles", CodePoints: []int{10877, 824}, Characters: []byte{0xe2, 0xa9, 0xbd, 0xcc, 0xb8}}, + "nless": {Name: "nless", CodePoints: []int{8814}, Characters: []byte{0xe2, 0x89, 0xae}}, + "nlsim": {Name: "nlsim", CodePoints: []int{8820}, Characters: []byte{0xe2, 0x89, 0xb4}}, + "nlt": {Name: "nlt", CodePoints: []int{8814}, Characters: []byte{0xe2, 0x89, 0xae}}, + "nltri": {Name: "nltri", CodePoints: []int{8938}, Characters: []byte{0xe2, 0x8b, 0xaa}}, + "nltrie": {Name: "nltrie", CodePoints: []int{8940}, Characters: []byte{0xe2, 0x8b, 0xac}}, + "nmid": {Name: "nmid", CodePoints: []int{8740}, Characters: []byte{0xe2, 0x88, 0xa4}}, + "nopf": {Name: "nopf", CodePoints: []int{120159}, Characters: []byte{0xf0, 0x9d, 0x95, 0x9f}}, + "not": {Name: "not", CodePoints: []int{172}, Characters: []byte{0xc2, 0xac}}, + "notin": {Name: "notin", CodePoints: []int{8713}, Characters: []byte{0xe2, 0x88, 0x89}}, + "notinE": {Name: "notinE", CodePoints: []int{8953, 824}, Characters: []byte{0xe2, 0x8b, 0xb9, 0xcc, 0xb8}}, + "notindot": {Name: "notindot", CodePoints: []int{8949, 824}, Characters: []byte{0xe2, 0x8b, 0xb5, 0xcc, 0xb8}}, + "notinva": {Name: "notinva", CodePoints: []int{8713}, Characters: []byte{0xe2, 0x88, 0x89}}, + "notinvb": {Name: "notinvb", CodePoints: []int{8951}, Characters: []byte{0xe2, 0x8b, 0xb7}}, + "notinvc": {Name: "notinvc", CodePoints: []int{8950}, Characters: []byte{0xe2, 0x8b, 0xb6}}, + "notni": {Name: "notni", CodePoints: []int{8716}, Characters: []byte{0xe2, 0x88, 0x8c}}, + "notniva": {Name: "notniva", CodePoints: []int{8716}, Characters: []byte{0xe2, 0x88, 0x8c}}, + "notnivb": {Name: "notnivb", CodePoints: []int{8958}, Characters: []byte{0xe2, 0x8b, 0xbe}}, + "notnivc": {Name: "notnivc", CodePoints: []int{8957}, Characters: []byte{0xe2, 0x8b, 0xbd}}, + "npar": {Name: "npar", CodePoints: []int{8742}, Characters: []byte{0xe2, 0x88, 0xa6}}, + "nparallel": {Name: "nparallel", CodePoints: []int{8742}, Characters: []byte{0xe2, 0x88, 0xa6}}, + "nparsl": {Name: "nparsl", CodePoints: []int{11005, 8421}, Characters: []byte{0xe2, 0xab, 0xbd, 0xe2, 0x83, 0xa5}}, + "npart": {Name: "npart", CodePoints: []int{8706, 824}, Characters: []byte{0xe2, 0x88, 0x82, 0xcc, 0xb8}}, + "npolint": {Name: "npolint", CodePoints: []int{10772}, Characters: []byte{0xe2, 0xa8, 0x94}}, + "npr": {Name: "npr", CodePoints: []int{8832}, Characters: []byte{0xe2, 0x8a, 0x80}}, + "nprcue": {Name: "nprcue", CodePoints: []int{8928}, Characters: []byte{0xe2, 0x8b, 0xa0}}, + "npre": {Name: "npre", CodePoints: []int{10927, 824}, Characters: []byte{0xe2, 0xaa, 0xaf, 0xcc, 0xb8}}, + "nprec": {Name: "nprec", CodePoints: []int{8832}, Characters: []byte{0xe2, 0x8a, 0x80}}, + "npreceq": {Name: "npreceq", CodePoints: []int{10927, 824}, Characters: []byte{0xe2, 0xaa, 0xaf, 0xcc, 0xb8}}, + "nrArr": {Name: "nrArr", CodePoints: []int{8655}, Characters: []byte{0xe2, 0x87, 0x8f}}, + "nrarr": {Name: "nrarr", CodePoints: []int{8603}, Characters: []byte{0xe2, 0x86, 0x9b}}, + "nrarrc": {Name: "nrarrc", CodePoints: []int{10547, 824}, Characters: []byte{0xe2, 0xa4, 0xb3, 0xcc, 0xb8}}, + "nrarrw": {Name: "nrarrw", CodePoints: []int{8605, 824}, Characters: []byte{0xe2, 0x86, 0x9d, 0xcc, 0xb8}}, + "nrightarrow": {Name: "nrightarrow", CodePoints: []int{8603}, Characters: []byte{0xe2, 0x86, 0x9b}}, + "nrtri": {Name: "nrtri", CodePoints: []int{8939}, Characters: []byte{0xe2, 0x8b, 0xab}}, + "nrtrie": {Name: "nrtrie", CodePoints: []int{8941}, Characters: []byte{0xe2, 0x8b, 0xad}}, + "nsc": {Name: "nsc", CodePoints: []int{8833}, Characters: []byte{0xe2, 0x8a, 0x81}}, + "nsccue": {Name: "nsccue", CodePoints: []int{8929}, Characters: []byte{0xe2, 0x8b, 0xa1}}, + "nsce": {Name: "nsce", CodePoints: []int{10928, 824}, Characters: []byte{0xe2, 0xaa, 0xb0, 0xcc, 0xb8}}, + "nscr": {Name: "nscr", CodePoints: []int{120003}, Characters: []byte{0xf0, 0x9d, 0x93, 0x83}}, + "nshortmid": {Name: "nshortmid", CodePoints: []int{8740}, Characters: []byte{0xe2, 0x88, 0xa4}}, + "nshortparallel": {Name: "nshortparallel", CodePoints: []int{8742}, Characters: []byte{0xe2, 0x88, 0xa6}}, + "nsim": {Name: "nsim", CodePoints: []int{8769}, Characters: []byte{0xe2, 0x89, 0x81}}, + "nsime": {Name: "nsime", CodePoints: []int{8772}, Characters: []byte{0xe2, 0x89, 0x84}}, + "nsimeq": {Name: "nsimeq", CodePoints: []int{8772}, Characters: []byte{0xe2, 0x89, 0x84}}, + "nsmid": {Name: "nsmid", CodePoints: []int{8740}, Characters: []byte{0xe2, 0x88, 0xa4}}, + "nspar": {Name: "nspar", CodePoints: []int{8742}, Characters: []byte{0xe2, 0x88, 0xa6}}, + "nsqsube": {Name: "nsqsube", CodePoints: []int{8930}, Characters: []byte{0xe2, 0x8b, 0xa2}}, + "nsqsupe": {Name: "nsqsupe", CodePoints: []int{8931}, Characters: []byte{0xe2, 0x8b, 0xa3}}, + "nsub": {Name: "nsub", CodePoints: []int{8836}, Characters: []byte{0xe2, 0x8a, 0x84}}, + "nsubE": {Name: "nsubE", CodePoints: []int{10949, 824}, Characters: []byte{0xe2, 0xab, 0x85, 0xcc, 0xb8}}, + "nsube": {Name: "nsube", CodePoints: []int{8840}, Characters: []byte{0xe2, 0x8a, 0x88}}, + "nsubset": {Name: "nsubset", CodePoints: []int{8834, 8402}, Characters: []byte{0xe2, 0x8a, 0x82, 0xe2, 0x83, 0x92}}, + "nsubseteq": {Name: "nsubseteq", CodePoints: []int{8840}, Characters: []byte{0xe2, 0x8a, 0x88}}, + "nsubseteqq": {Name: "nsubseteqq", CodePoints: []int{10949, 824}, Characters: []byte{0xe2, 0xab, 0x85, 0xcc, 0xb8}}, + "nsucc": {Name: "nsucc", CodePoints: []int{8833}, Characters: []byte{0xe2, 0x8a, 0x81}}, + "nsucceq": {Name: "nsucceq", CodePoints: []int{10928, 824}, Characters: []byte{0xe2, 0xaa, 0xb0, 0xcc, 0xb8}}, + "nsup": {Name: "nsup", CodePoints: []int{8837}, Characters: []byte{0xe2, 0x8a, 0x85}}, + "nsupE": {Name: "nsupE", CodePoints: []int{10950, 824}, Characters: []byte{0xe2, 0xab, 0x86, 0xcc, 0xb8}}, + "nsupe": {Name: "nsupe", CodePoints: []int{8841}, Characters: []byte{0xe2, 0x8a, 0x89}}, + "nsupset": {Name: "nsupset", CodePoints: []int{8835, 8402}, Characters: []byte{0xe2, 0x8a, 0x83, 0xe2, 0x83, 0x92}}, + "nsupseteq": {Name: "nsupseteq", CodePoints: []int{8841}, Characters: []byte{0xe2, 0x8a, 0x89}}, + "nsupseteqq": {Name: "nsupseteqq", CodePoints: []int{10950, 824}, Characters: []byte{0xe2, 0xab, 0x86, 0xcc, 0xb8}}, + "ntgl": {Name: "ntgl", CodePoints: []int{8825}, Characters: []byte{0xe2, 0x89, 0xb9}}, + "ntilde": {Name: "ntilde", CodePoints: []int{241}, Characters: []byte{0xc3, 0xb1}}, + "ntlg": {Name: "ntlg", CodePoints: []int{8824}, Characters: []byte{0xe2, 0x89, 0xb8}}, + "ntriangleleft": {Name: "ntriangleleft", CodePoints: []int{8938}, Characters: []byte{0xe2, 0x8b, 0xaa}}, + "ntrianglelefteq": {Name: "ntrianglelefteq", CodePoints: []int{8940}, Characters: []byte{0xe2, 0x8b, 0xac}}, + "ntriangleright": {Name: "ntriangleright", CodePoints: []int{8939}, Characters: []byte{0xe2, 0x8b, 0xab}}, + "ntrianglerighteq": {Name: "ntrianglerighteq", CodePoints: []int{8941}, Characters: []byte{0xe2, 0x8b, 0xad}}, + "nu": {Name: "nu", CodePoints: []int{957}, Characters: []byte{0xce, 0xbd}}, + "num": {Name: "num", CodePoints: []int{35}, Characters: []byte{0x23}}, + "numero": {Name: "numero", CodePoints: []int{8470}, Characters: []byte{0xe2, 0x84, 0x96}}, + "numsp": {Name: "numsp", CodePoints: []int{8199}, Characters: []byte{0xe2, 0x80, 0x87}}, + "nvDash": {Name: "nvDash", CodePoints: []int{8877}, Characters: []byte{0xe2, 0x8a, 0xad}}, + "nvHarr": {Name: "nvHarr", CodePoints: []int{10500}, Characters: []byte{0xe2, 0xa4, 0x84}}, + "nvap": {Name: "nvap", CodePoints: []int{8781, 8402}, Characters: []byte{0xe2, 0x89, 0x8d, 0xe2, 0x83, 0x92}}, + "nvdash": {Name: "nvdash", CodePoints: []int{8876}, Characters: []byte{0xe2, 0x8a, 0xac}}, + "nvge": {Name: "nvge", CodePoints: []int{8805, 8402}, Characters: []byte{0xe2, 0x89, 0xa5, 0xe2, 0x83, 0x92}}, + "nvgt": {Name: "nvgt", CodePoints: []int{62, 8402}, Characters: []byte{0x3e, 0xe2, 0x83, 0x92}}, + "nvinfin": {Name: "nvinfin", CodePoints: []int{10718}, Characters: []byte{0xe2, 0xa7, 0x9e}}, + "nvlArr": {Name: "nvlArr", CodePoints: []int{10498}, Characters: []byte{0xe2, 0xa4, 0x82}}, + "nvle": {Name: "nvle", CodePoints: []int{8804, 8402}, Characters: []byte{0xe2, 0x89, 0xa4, 0xe2, 0x83, 0x92}}, + "nvlt": {Name: "nvlt", CodePoints: []int{60, 8402}, Characters: []byte{0x3c, 0xe2, 0x83, 0x92}}, + "nvltrie": {Name: "nvltrie", CodePoints: []int{8884, 8402}, Characters: []byte{0xe2, 0x8a, 0xb4, 0xe2, 0x83, 0x92}}, + "nvrArr": {Name: "nvrArr", CodePoints: []int{10499}, Characters: []byte{0xe2, 0xa4, 0x83}}, + "nvrtrie": {Name: "nvrtrie", CodePoints: []int{8885, 8402}, Characters: []byte{0xe2, 0x8a, 0xb5, 0xe2, 0x83, 0x92}}, + "nvsim": {Name: "nvsim", CodePoints: []int{8764, 8402}, Characters: []byte{0xe2, 0x88, 0xbc, 0xe2, 0x83, 0x92}}, + "nwArr": {Name: "nwArr", CodePoints: []int{8662}, Characters: []byte{0xe2, 0x87, 0x96}}, + "nwarhk": {Name: "nwarhk", CodePoints: []int{10531}, Characters: []byte{0xe2, 0xa4, 0xa3}}, + "nwarr": {Name: "nwarr", CodePoints: []int{8598}, Characters: []byte{0xe2, 0x86, 0x96}}, + "nwarrow": {Name: "nwarrow", CodePoints: []int{8598}, Characters: []byte{0xe2, 0x86, 0x96}}, + "nwnear": {Name: "nwnear", CodePoints: []int{10535}, Characters: []byte{0xe2, 0xa4, 0xa7}}, + "oS": {Name: "oS", CodePoints: []int{9416}, Characters: []byte{0xe2, 0x93, 0x88}}, + "oacute": {Name: "oacute", CodePoints: []int{243}, Characters: []byte{0xc3, 0xb3}}, + "oast": {Name: "oast", CodePoints: []int{8859}, Characters: []byte{0xe2, 0x8a, 0x9b}}, + "ocir": {Name: "ocir", CodePoints: []int{8858}, Characters: []byte{0xe2, 0x8a, 0x9a}}, + "ocirc": {Name: "ocirc", CodePoints: []int{244}, Characters: []byte{0xc3, 0xb4}}, + "ocy": {Name: "ocy", CodePoints: []int{1086}, Characters: []byte{0xd0, 0xbe}}, + "odash": {Name: "odash", CodePoints: []int{8861}, Characters: []byte{0xe2, 0x8a, 0x9d}}, + "odblac": {Name: "odblac", CodePoints: []int{337}, Characters: []byte{0xc5, 0x91}}, + "odiv": {Name: "odiv", CodePoints: []int{10808}, Characters: []byte{0xe2, 0xa8, 0xb8}}, + "odot": {Name: "odot", CodePoints: []int{8857}, Characters: []byte{0xe2, 0x8a, 0x99}}, + "odsold": {Name: "odsold", CodePoints: []int{10684}, Characters: []byte{0xe2, 0xa6, 0xbc}}, + "oelig": {Name: "oelig", CodePoints: []int{339}, Characters: []byte{0xc5, 0x93}}, + "ofcir": {Name: "ofcir", CodePoints: []int{10687}, Characters: []byte{0xe2, 0xa6, 0xbf}}, + "ofr": {Name: "ofr", CodePoints: []int{120108}, Characters: []byte{0xf0, 0x9d, 0x94, 0xac}}, + "ogon": {Name: "ogon", CodePoints: []int{731}, Characters: []byte{0xcb, 0x9b}}, + "ograve": {Name: "ograve", CodePoints: []int{242}, Characters: []byte{0xc3, 0xb2}}, + "ogt": {Name: "ogt", CodePoints: []int{10689}, Characters: []byte{0xe2, 0xa7, 0x81}}, + "ohbar": {Name: "ohbar", CodePoints: []int{10677}, Characters: []byte{0xe2, 0xa6, 0xb5}}, + "ohm": {Name: "ohm", CodePoints: []int{937}, Characters: []byte{0xce, 0xa9}}, + "oint": {Name: "oint", CodePoints: []int{8750}, Characters: []byte{0xe2, 0x88, 0xae}}, + "olarr": {Name: "olarr", CodePoints: []int{8634}, Characters: []byte{0xe2, 0x86, 0xba}}, + "olcir": {Name: "olcir", CodePoints: []int{10686}, Characters: []byte{0xe2, 0xa6, 0xbe}}, + "olcross": {Name: "olcross", CodePoints: []int{10683}, Characters: []byte{0xe2, 0xa6, 0xbb}}, + "oline": {Name: "oline", CodePoints: []int{8254}, Characters: []byte{0xe2, 0x80, 0xbe}}, + "olt": {Name: "olt", CodePoints: []int{10688}, Characters: []byte{0xe2, 0xa7, 0x80}}, + "omacr": {Name: "omacr", CodePoints: []int{333}, Characters: []byte{0xc5, 0x8d}}, + "omega": {Name: "omega", CodePoints: []int{969}, Characters: []byte{0xcf, 0x89}}, + "omicron": {Name: "omicron", CodePoints: []int{959}, Characters: []byte{0xce, 0xbf}}, + "omid": {Name: "omid", CodePoints: []int{10678}, Characters: []byte{0xe2, 0xa6, 0xb6}}, + "ominus": {Name: "ominus", CodePoints: []int{8854}, Characters: []byte{0xe2, 0x8a, 0x96}}, + "oopf": {Name: "oopf", CodePoints: []int{120160}, Characters: []byte{0xf0, 0x9d, 0x95, 0xa0}}, + "opar": {Name: "opar", CodePoints: []int{10679}, Characters: []byte{0xe2, 0xa6, 0xb7}}, + "operp": {Name: "operp", CodePoints: []int{10681}, Characters: []byte{0xe2, 0xa6, 0xb9}}, + "oplus": {Name: "oplus", CodePoints: []int{8853}, Characters: []byte{0xe2, 0x8a, 0x95}}, + "or": {Name: "or", CodePoints: []int{8744}, Characters: []byte{0xe2, 0x88, 0xa8}}, + "orarr": {Name: "orarr", CodePoints: []int{8635}, Characters: []byte{0xe2, 0x86, 0xbb}}, + "ord": {Name: "ord", CodePoints: []int{10845}, Characters: []byte{0xe2, 0xa9, 0x9d}}, + "order": {Name: "order", CodePoints: []int{8500}, Characters: []byte{0xe2, 0x84, 0xb4}}, + "orderof": {Name: "orderof", CodePoints: []int{8500}, Characters: []byte{0xe2, 0x84, 0xb4}}, + "ordf": {Name: "ordf", CodePoints: []int{170}, Characters: []byte{0xc2, 0xaa}}, + "ordm": {Name: "ordm", CodePoints: []int{186}, Characters: []byte{0xc2, 0xba}}, + "origof": {Name: "origof", CodePoints: []int{8886}, Characters: []byte{0xe2, 0x8a, 0xb6}}, + "oror": {Name: "oror", CodePoints: []int{10838}, Characters: []byte{0xe2, 0xa9, 0x96}}, + "orslope": {Name: "orslope", CodePoints: []int{10839}, Characters: []byte{0xe2, 0xa9, 0x97}}, + "orv": {Name: "orv", CodePoints: []int{10843}, Characters: []byte{0xe2, 0xa9, 0x9b}}, + "oscr": {Name: "oscr", CodePoints: []int{8500}, Characters: []byte{0xe2, 0x84, 0xb4}}, + "oslash": {Name: "oslash", CodePoints: []int{248}, Characters: []byte{0xc3, 0xb8}}, + "osol": {Name: "osol", CodePoints: []int{8856}, Characters: []byte{0xe2, 0x8a, 0x98}}, + "otilde": {Name: "otilde", CodePoints: []int{245}, Characters: []byte{0xc3, 0xb5}}, + "otimes": {Name: "otimes", CodePoints: []int{8855}, Characters: []byte{0xe2, 0x8a, 0x97}}, + "otimesas": {Name: "otimesas", CodePoints: []int{10806}, Characters: []byte{0xe2, 0xa8, 0xb6}}, + "ouml": {Name: "ouml", CodePoints: []int{246}, Characters: []byte{0xc3, 0xb6}}, + "ovbar": {Name: "ovbar", CodePoints: []int{9021}, Characters: []byte{0xe2, 0x8c, 0xbd}}, + "par": {Name: "par", CodePoints: []int{8741}, Characters: []byte{0xe2, 0x88, 0xa5}}, + "para": {Name: "para", CodePoints: []int{182}, Characters: []byte{0xc2, 0xb6}}, + "parallel": {Name: "parallel", CodePoints: []int{8741}, Characters: []byte{0xe2, 0x88, 0xa5}}, + "parsim": {Name: "parsim", CodePoints: []int{10995}, Characters: []byte{0xe2, 0xab, 0xb3}}, + "parsl": {Name: "parsl", CodePoints: []int{11005}, Characters: []byte{0xe2, 0xab, 0xbd}}, + "part": {Name: "part", CodePoints: []int{8706}, Characters: []byte{0xe2, 0x88, 0x82}}, + "pcy": {Name: "pcy", CodePoints: []int{1087}, Characters: []byte{0xd0, 0xbf}}, + "percnt": {Name: "percnt", CodePoints: []int{37}, Characters: []byte{0x25}}, + "period": {Name: "period", CodePoints: []int{46}, Characters: []byte{0x2e}}, + "permil": {Name: "permil", CodePoints: []int{8240}, Characters: []byte{0xe2, 0x80, 0xb0}}, + "perp": {Name: "perp", CodePoints: []int{8869}, Characters: []byte{0xe2, 0x8a, 0xa5}}, + "pertenk": {Name: "pertenk", CodePoints: []int{8241}, Characters: []byte{0xe2, 0x80, 0xb1}}, + "pfr": {Name: "pfr", CodePoints: []int{120109}, Characters: []byte{0xf0, 0x9d, 0x94, 0xad}}, + "phi": {Name: "phi", CodePoints: []int{966}, Characters: []byte{0xcf, 0x86}}, + "phiv": {Name: "phiv", CodePoints: []int{981}, Characters: []byte{0xcf, 0x95}}, + "phmmat": {Name: "phmmat", CodePoints: []int{8499}, Characters: []byte{0xe2, 0x84, 0xb3}}, + "phone": {Name: "phone", CodePoints: []int{9742}, Characters: []byte{0xe2, 0x98, 0x8e}}, + "pi": {Name: "pi", CodePoints: []int{960}, Characters: []byte{0xcf, 0x80}}, + "pitchfork": {Name: "pitchfork", CodePoints: []int{8916}, Characters: []byte{0xe2, 0x8b, 0x94}}, + "piv": {Name: "piv", CodePoints: []int{982}, Characters: []byte{0xcf, 0x96}}, + "planck": {Name: "planck", CodePoints: []int{8463}, Characters: []byte{0xe2, 0x84, 0x8f}}, + "planckh": {Name: "planckh", CodePoints: []int{8462}, Characters: []byte{0xe2, 0x84, 0x8e}}, + "plankv": {Name: "plankv", CodePoints: []int{8463}, Characters: []byte{0xe2, 0x84, 0x8f}}, + "plus": {Name: "plus", CodePoints: []int{43}, Characters: []byte{0x2b}}, + "plusacir": {Name: "plusacir", CodePoints: []int{10787}, Characters: []byte{0xe2, 0xa8, 0xa3}}, + "plusb": {Name: "plusb", CodePoints: []int{8862}, Characters: []byte{0xe2, 0x8a, 0x9e}}, + "pluscir": {Name: "pluscir", CodePoints: []int{10786}, Characters: []byte{0xe2, 0xa8, 0xa2}}, + "plusdo": {Name: "plusdo", CodePoints: []int{8724}, Characters: []byte{0xe2, 0x88, 0x94}}, + "plusdu": {Name: "plusdu", CodePoints: []int{10789}, Characters: []byte{0xe2, 0xa8, 0xa5}}, + "pluse": {Name: "pluse", CodePoints: []int{10866}, Characters: []byte{0xe2, 0xa9, 0xb2}}, + "plusmn": {Name: "plusmn", CodePoints: []int{177}, Characters: []byte{0xc2, 0xb1}}, + "plussim": {Name: "plussim", CodePoints: []int{10790}, Characters: []byte{0xe2, 0xa8, 0xa6}}, + "plustwo": {Name: "plustwo", CodePoints: []int{10791}, Characters: []byte{0xe2, 0xa8, 0xa7}}, + "pm": {Name: "pm", CodePoints: []int{177}, Characters: []byte{0xc2, 0xb1}}, + "pointint": {Name: "pointint", CodePoints: []int{10773}, Characters: []byte{0xe2, 0xa8, 0x95}}, + "popf": {Name: "popf", CodePoints: []int{120161}, Characters: []byte{0xf0, 0x9d, 0x95, 0xa1}}, + "pound": {Name: "pound", CodePoints: []int{163}, Characters: []byte{0xc2, 0xa3}}, + "pr": {Name: "pr", CodePoints: []int{8826}, Characters: []byte{0xe2, 0x89, 0xba}}, + "prE": {Name: "prE", CodePoints: []int{10931}, Characters: []byte{0xe2, 0xaa, 0xb3}}, + "prap": {Name: "prap", CodePoints: []int{10935}, Characters: []byte{0xe2, 0xaa, 0xb7}}, + "prcue": {Name: "prcue", CodePoints: []int{8828}, Characters: []byte{0xe2, 0x89, 0xbc}}, + "pre": {Name: "pre", CodePoints: []int{10927}, Characters: []byte{0xe2, 0xaa, 0xaf}}, + "prec": {Name: "prec", CodePoints: []int{8826}, Characters: []byte{0xe2, 0x89, 0xba}}, + "precapprox": {Name: "precapprox", CodePoints: []int{10935}, Characters: []byte{0xe2, 0xaa, 0xb7}}, + "preccurlyeq": {Name: "preccurlyeq", CodePoints: []int{8828}, Characters: []byte{0xe2, 0x89, 0xbc}}, + "preceq": {Name: "preceq", CodePoints: []int{10927}, Characters: []byte{0xe2, 0xaa, 0xaf}}, + "precnapprox": {Name: "precnapprox", CodePoints: []int{10937}, Characters: []byte{0xe2, 0xaa, 0xb9}}, + "precneqq": {Name: "precneqq", CodePoints: []int{10933}, Characters: []byte{0xe2, 0xaa, 0xb5}}, + "precnsim": {Name: "precnsim", CodePoints: []int{8936}, Characters: []byte{0xe2, 0x8b, 0xa8}}, + "precsim": {Name: "precsim", CodePoints: []int{8830}, Characters: []byte{0xe2, 0x89, 0xbe}}, + "prime": {Name: "prime", CodePoints: []int{8242}, Characters: []byte{0xe2, 0x80, 0xb2}}, + "primes": {Name: "primes", CodePoints: []int{8473}, Characters: []byte{0xe2, 0x84, 0x99}}, + "prnE": {Name: "prnE", CodePoints: []int{10933}, Characters: []byte{0xe2, 0xaa, 0xb5}}, + "prnap": {Name: "prnap", CodePoints: []int{10937}, Characters: []byte{0xe2, 0xaa, 0xb9}}, + "prnsim": {Name: "prnsim", CodePoints: []int{8936}, Characters: []byte{0xe2, 0x8b, 0xa8}}, + "prod": {Name: "prod", CodePoints: []int{8719}, Characters: []byte{0xe2, 0x88, 0x8f}}, + "profalar": {Name: "profalar", CodePoints: []int{9006}, Characters: []byte{0xe2, 0x8c, 0xae}}, + "profline": {Name: "profline", CodePoints: []int{8978}, Characters: []byte{0xe2, 0x8c, 0x92}}, + "profsurf": {Name: "profsurf", CodePoints: []int{8979}, Characters: []byte{0xe2, 0x8c, 0x93}}, + "prop": {Name: "prop", CodePoints: []int{8733}, Characters: []byte{0xe2, 0x88, 0x9d}}, + "propto": {Name: "propto", CodePoints: []int{8733}, Characters: []byte{0xe2, 0x88, 0x9d}}, + "prsim": {Name: "prsim", CodePoints: []int{8830}, Characters: []byte{0xe2, 0x89, 0xbe}}, + "prurel": {Name: "prurel", CodePoints: []int{8880}, Characters: []byte{0xe2, 0x8a, 0xb0}}, + "pscr": {Name: "pscr", CodePoints: []int{120005}, Characters: []byte{0xf0, 0x9d, 0x93, 0x85}}, + "psi": {Name: "psi", CodePoints: []int{968}, Characters: []byte{0xcf, 0x88}}, + "puncsp": {Name: "puncsp", CodePoints: []int{8200}, Characters: []byte{0xe2, 0x80, 0x88}}, + "qfr": {Name: "qfr", CodePoints: []int{120110}, Characters: []byte{0xf0, 0x9d, 0x94, 0xae}}, + "qint": {Name: "qint", CodePoints: []int{10764}, Characters: []byte{0xe2, 0xa8, 0x8c}}, + "qopf": {Name: "qopf", CodePoints: []int{120162}, Characters: []byte{0xf0, 0x9d, 0x95, 0xa2}}, + "qprime": {Name: "qprime", CodePoints: []int{8279}, Characters: []byte{0xe2, 0x81, 0x97}}, + "qscr": {Name: "qscr", CodePoints: []int{120006}, Characters: []byte{0xf0, 0x9d, 0x93, 0x86}}, + "quaternions": {Name: "quaternions", CodePoints: []int{8461}, Characters: []byte{0xe2, 0x84, 0x8d}}, + "quatint": {Name: "quatint", CodePoints: []int{10774}, Characters: []byte{0xe2, 0xa8, 0x96}}, + "quest": {Name: "quest", CodePoints: []int{63}, Characters: []byte{0x3f}}, + "questeq": {Name: "questeq", CodePoints: []int{8799}, Characters: []byte{0xe2, 0x89, 0x9f}}, + "quot": {Name: "quot", CodePoints: []int{34}, Characters: []byte{0x22}}, + "rAarr": {Name: "rAarr", CodePoints: []int{8667}, Characters: []byte{0xe2, 0x87, 0x9b}}, + "rArr": {Name: "rArr", CodePoints: []int{8658}, Characters: []byte{0xe2, 0x87, 0x92}}, + "rAtail": {Name: "rAtail", CodePoints: []int{10524}, Characters: []byte{0xe2, 0xa4, 0x9c}}, + "rBarr": {Name: "rBarr", CodePoints: []int{10511}, Characters: []byte{0xe2, 0xa4, 0x8f}}, + "rHar": {Name: "rHar", CodePoints: []int{10596}, Characters: []byte{0xe2, 0xa5, 0xa4}}, + "race": {Name: "race", CodePoints: []int{8765, 817}, Characters: []byte{0xe2, 0x88, 0xbd, 0xcc, 0xb1}}, + "racute": {Name: "racute", CodePoints: []int{341}, Characters: []byte{0xc5, 0x95}}, + "radic": {Name: "radic", CodePoints: []int{8730}, Characters: []byte{0xe2, 0x88, 0x9a}}, + "raemptyv": {Name: "raemptyv", CodePoints: []int{10675}, Characters: []byte{0xe2, 0xa6, 0xb3}}, + "rang": {Name: "rang", CodePoints: []int{10217}, Characters: []byte{0xe2, 0x9f, 0xa9}}, + "rangd": {Name: "rangd", CodePoints: []int{10642}, Characters: []byte{0xe2, 0xa6, 0x92}}, + "range": {Name: "range", CodePoints: []int{10661}, Characters: []byte{0xe2, 0xa6, 0xa5}}, + "rangle": {Name: "rangle", CodePoints: []int{10217}, Characters: []byte{0xe2, 0x9f, 0xa9}}, + "raquo": {Name: "raquo", CodePoints: []int{187}, Characters: []byte{0xc2, 0xbb}}, + "rarr": {Name: "rarr", CodePoints: []int{8594}, Characters: []byte{0xe2, 0x86, 0x92}}, + "rarrap": {Name: "rarrap", CodePoints: []int{10613}, Characters: []byte{0xe2, 0xa5, 0xb5}}, + "rarrb": {Name: "rarrb", CodePoints: []int{8677}, Characters: []byte{0xe2, 0x87, 0xa5}}, + "rarrbfs": {Name: "rarrbfs", CodePoints: []int{10528}, Characters: []byte{0xe2, 0xa4, 0xa0}}, + "rarrc": {Name: "rarrc", CodePoints: []int{10547}, Characters: []byte{0xe2, 0xa4, 0xb3}}, + "rarrfs": {Name: "rarrfs", CodePoints: []int{10526}, Characters: []byte{0xe2, 0xa4, 0x9e}}, + "rarrhk": {Name: "rarrhk", CodePoints: []int{8618}, Characters: []byte{0xe2, 0x86, 0xaa}}, + "rarrlp": {Name: "rarrlp", CodePoints: []int{8620}, Characters: []byte{0xe2, 0x86, 0xac}}, + "rarrpl": {Name: "rarrpl", CodePoints: []int{10565}, Characters: []byte{0xe2, 0xa5, 0x85}}, + "rarrsim": {Name: "rarrsim", CodePoints: []int{10612}, Characters: []byte{0xe2, 0xa5, 0xb4}}, + "rarrtl": {Name: "rarrtl", CodePoints: []int{8611}, Characters: []byte{0xe2, 0x86, 0xa3}}, + "rarrw": {Name: "rarrw", CodePoints: []int{8605}, Characters: []byte{0xe2, 0x86, 0x9d}}, + "ratail": {Name: "ratail", CodePoints: []int{10522}, Characters: []byte{0xe2, 0xa4, 0x9a}}, + "ratio": {Name: "ratio", CodePoints: []int{8758}, Characters: []byte{0xe2, 0x88, 0xb6}}, + "rationals": {Name: "rationals", CodePoints: []int{8474}, Characters: []byte{0xe2, 0x84, 0x9a}}, + "rbarr": {Name: "rbarr", CodePoints: []int{10509}, Characters: []byte{0xe2, 0xa4, 0x8d}}, + "rbbrk": {Name: "rbbrk", CodePoints: []int{10099}, Characters: []byte{0xe2, 0x9d, 0xb3}}, + "rbrace": {Name: "rbrace", CodePoints: []int{125}, Characters: []byte{0x7d}}, + "rbrack": {Name: "rbrack", CodePoints: []int{93}, Characters: []byte{0x5d}}, + "rbrke": {Name: "rbrke", CodePoints: []int{10636}, Characters: []byte{0xe2, 0xa6, 0x8c}}, + "rbrksld": {Name: "rbrksld", CodePoints: []int{10638}, Characters: []byte{0xe2, 0xa6, 0x8e}}, + "rbrkslu": {Name: "rbrkslu", CodePoints: []int{10640}, Characters: []byte{0xe2, 0xa6, 0x90}}, + "rcaron": {Name: "rcaron", CodePoints: []int{345}, Characters: []byte{0xc5, 0x99}}, + "rcedil": {Name: "rcedil", CodePoints: []int{343}, Characters: []byte{0xc5, 0x97}}, + "rceil": {Name: "rceil", CodePoints: []int{8969}, Characters: []byte{0xe2, 0x8c, 0x89}}, + "rcub": {Name: "rcub", CodePoints: []int{125}, Characters: []byte{0x7d}}, + "rcy": {Name: "rcy", CodePoints: []int{1088}, Characters: []byte{0xd1, 0x80}}, + "rdca": {Name: "rdca", CodePoints: []int{10551}, Characters: []byte{0xe2, 0xa4, 0xb7}}, + "rdldhar": {Name: "rdldhar", CodePoints: []int{10601}, Characters: []byte{0xe2, 0xa5, 0xa9}}, + "rdquo": {Name: "rdquo", CodePoints: []int{8221}, Characters: []byte{0xe2, 0x80, 0x9d}}, + "rdquor": {Name: "rdquor", CodePoints: []int{8221}, Characters: []byte{0xe2, 0x80, 0x9d}}, + "rdsh": {Name: "rdsh", CodePoints: []int{8627}, Characters: []byte{0xe2, 0x86, 0xb3}}, + "real": {Name: "real", CodePoints: []int{8476}, Characters: []byte{0xe2, 0x84, 0x9c}}, + "realine": {Name: "realine", CodePoints: []int{8475}, Characters: []byte{0xe2, 0x84, 0x9b}}, + "realpart": {Name: "realpart", CodePoints: []int{8476}, Characters: []byte{0xe2, 0x84, 0x9c}}, + "reals": {Name: "reals", CodePoints: []int{8477}, Characters: []byte{0xe2, 0x84, 0x9d}}, + "rect": {Name: "rect", CodePoints: []int{9645}, Characters: []byte{0xe2, 0x96, 0xad}}, + "reg": {Name: "reg", CodePoints: []int{174}, Characters: []byte{0xc2, 0xae}}, + "rfisht": {Name: "rfisht", CodePoints: []int{10621}, Characters: []byte{0xe2, 0xa5, 0xbd}}, + "rfloor": {Name: "rfloor", CodePoints: []int{8971}, Characters: []byte{0xe2, 0x8c, 0x8b}}, + "rfr": {Name: "rfr", CodePoints: []int{120111}, Characters: []byte{0xf0, 0x9d, 0x94, 0xaf}}, + "rhard": {Name: "rhard", CodePoints: []int{8641}, Characters: []byte{0xe2, 0x87, 0x81}}, + "rharu": {Name: "rharu", CodePoints: []int{8640}, Characters: []byte{0xe2, 0x87, 0x80}}, + "rharul": {Name: "rharul", CodePoints: []int{10604}, Characters: []byte{0xe2, 0xa5, 0xac}}, + "rho": {Name: "rho", CodePoints: []int{961}, Characters: []byte{0xcf, 0x81}}, + "rhov": {Name: "rhov", CodePoints: []int{1009}, Characters: []byte{0xcf, 0xb1}}, + "rightarrow": {Name: "rightarrow", CodePoints: []int{8594}, Characters: []byte{0xe2, 0x86, 0x92}}, + "rightarrowtail": {Name: "rightarrowtail", CodePoints: []int{8611}, Characters: []byte{0xe2, 0x86, 0xa3}}, + "rightharpoondown": {Name: "rightharpoondown", CodePoints: []int{8641}, Characters: []byte{0xe2, 0x87, 0x81}}, + "rightharpoonup": {Name: "rightharpoonup", CodePoints: []int{8640}, Characters: []byte{0xe2, 0x87, 0x80}}, + "rightleftarrows": {Name: "rightleftarrows", CodePoints: []int{8644}, Characters: []byte{0xe2, 0x87, 0x84}}, + "rightleftharpoons": {Name: "rightleftharpoons", CodePoints: []int{8652}, Characters: []byte{0xe2, 0x87, 0x8c}}, + "rightrightarrows": {Name: "rightrightarrows", CodePoints: []int{8649}, Characters: []byte{0xe2, 0x87, 0x89}}, + "rightsquigarrow": {Name: "rightsquigarrow", CodePoints: []int{8605}, Characters: []byte{0xe2, 0x86, 0x9d}}, + "rightthreetimes": {Name: "rightthreetimes", CodePoints: []int{8908}, Characters: []byte{0xe2, 0x8b, 0x8c}}, + "ring": {Name: "ring", CodePoints: []int{730}, Characters: []byte{0xcb, 0x9a}}, + "risingdotseq": {Name: "risingdotseq", CodePoints: []int{8787}, Characters: []byte{0xe2, 0x89, 0x93}}, + "rlarr": {Name: "rlarr", CodePoints: []int{8644}, Characters: []byte{0xe2, 0x87, 0x84}}, + "rlhar": {Name: "rlhar", CodePoints: []int{8652}, Characters: []byte{0xe2, 0x87, 0x8c}}, + "rlm": {Name: "rlm", CodePoints: []int{8207}, Characters: []byte{0xe2, 0x80, 0x8f}}, + "rmoust": {Name: "rmoust", CodePoints: []int{9137}, Characters: []byte{0xe2, 0x8e, 0xb1}}, + "rmoustache": {Name: "rmoustache", CodePoints: []int{9137}, Characters: []byte{0xe2, 0x8e, 0xb1}}, + "rnmid": {Name: "rnmid", CodePoints: []int{10990}, Characters: []byte{0xe2, 0xab, 0xae}}, + "roang": {Name: "roang", CodePoints: []int{10221}, Characters: []byte{0xe2, 0x9f, 0xad}}, + "roarr": {Name: "roarr", CodePoints: []int{8702}, Characters: []byte{0xe2, 0x87, 0xbe}}, + "robrk": {Name: "robrk", CodePoints: []int{10215}, Characters: []byte{0xe2, 0x9f, 0xa7}}, + "ropar": {Name: "ropar", CodePoints: []int{10630}, Characters: []byte{0xe2, 0xa6, 0x86}}, + "ropf": {Name: "ropf", CodePoints: []int{120163}, Characters: []byte{0xf0, 0x9d, 0x95, 0xa3}}, + "roplus": {Name: "roplus", CodePoints: []int{10798}, Characters: []byte{0xe2, 0xa8, 0xae}}, + "rotimes": {Name: "rotimes", CodePoints: []int{10805}, Characters: []byte{0xe2, 0xa8, 0xb5}}, + "rpar": {Name: "rpar", CodePoints: []int{41}, Characters: []byte{0x29}}, + "rpargt": {Name: "rpargt", CodePoints: []int{10644}, Characters: []byte{0xe2, 0xa6, 0x94}}, + "rppolint": {Name: "rppolint", CodePoints: []int{10770}, Characters: []byte{0xe2, 0xa8, 0x92}}, + "rrarr": {Name: "rrarr", CodePoints: []int{8649}, Characters: []byte{0xe2, 0x87, 0x89}}, + "rsaquo": {Name: "rsaquo", CodePoints: []int{8250}, Characters: []byte{0xe2, 0x80, 0xba}}, + "rscr": {Name: "rscr", CodePoints: []int{120007}, Characters: []byte{0xf0, 0x9d, 0x93, 0x87}}, + "rsh": {Name: "rsh", CodePoints: []int{8625}, Characters: []byte{0xe2, 0x86, 0xb1}}, + "rsqb": {Name: "rsqb", CodePoints: []int{93}, Characters: []byte{0x5d}}, + "rsquo": {Name: "rsquo", CodePoints: []int{8217}, Characters: []byte{0xe2, 0x80, 0x99}}, + "rsquor": {Name: "rsquor", CodePoints: []int{8217}, Characters: []byte{0xe2, 0x80, 0x99}}, + "rthree": {Name: "rthree", CodePoints: []int{8908}, Characters: []byte{0xe2, 0x8b, 0x8c}}, + "rtimes": {Name: "rtimes", CodePoints: []int{8906}, Characters: []byte{0xe2, 0x8b, 0x8a}}, + "rtri": {Name: "rtri", CodePoints: []int{9657}, Characters: []byte{0xe2, 0x96, 0xb9}}, + "rtrie": {Name: "rtrie", CodePoints: []int{8885}, Characters: []byte{0xe2, 0x8a, 0xb5}}, + "rtrif": {Name: "rtrif", CodePoints: []int{9656}, Characters: []byte{0xe2, 0x96, 0xb8}}, + "rtriltri": {Name: "rtriltri", CodePoints: []int{10702}, Characters: []byte{0xe2, 0xa7, 0x8e}}, + "ruluhar": {Name: "ruluhar", CodePoints: []int{10600}, Characters: []byte{0xe2, 0xa5, 0xa8}}, + "rx": {Name: "rx", CodePoints: []int{8478}, Characters: []byte{0xe2, 0x84, 0x9e}}, + "sacute": {Name: "sacute", CodePoints: []int{347}, Characters: []byte{0xc5, 0x9b}}, + "sbquo": {Name: "sbquo", CodePoints: []int{8218}, Characters: []byte{0xe2, 0x80, 0x9a}}, + "sc": {Name: "sc", CodePoints: []int{8827}, Characters: []byte{0xe2, 0x89, 0xbb}}, + "scE": {Name: "scE", CodePoints: []int{10932}, Characters: []byte{0xe2, 0xaa, 0xb4}}, + "scap": {Name: "scap", CodePoints: []int{10936}, Characters: []byte{0xe2, 0xaa, 0xb8}}, + "scaron": {Name: "scaron", CodePoints: []int{353}, Characters: []byte{0xc5, 0xa1}}, + "sccue": {Name: "sccue", CodePoints: []int{8829}, Characters: []byte{0xe2, 0x89, 0xbd}}, + "sce": {Name: "sce", CodePoints: []int{10928}, Characters: []byte{0xe2, 0xaa, 0xb0}}, + "scedil": {Name: "scedil", CodePoints: []int{351}, Characters: []byte{0xc5, 0x9f}}, + "scirc": {Name: "scirc", CodePoints: []int{349}, Characters: []byte{0xc5, 0x9d}}, + "scnE": {Name: "scnE", CodePoints: []int{10934}, Characters: []byte{0xe2, 0xaa, 0xb6}}, + "scnap": {Name: "scnap", CodePoints: []int{10938}, Characters: []byte{0xe2, 0xaa, 0xba}}, + "scnsim": {Name: "scnsim", CodePoints: []int{8937}, Characters: []byte{0xe2, 0x8b, 0xa9}}, + "scpolint": {Name: "scpolint", CodePoints: []int{10771}, Characters: []byte{0xe2, 0xa8, 0x93}}, + "scsim": {Name: "scsim", CodePoints: []int{8831}, Characters: []byte{0xe2, 0x89, 0xbf}}, + "scy": {Name: "scy", CodePoints: []int{1089}, Characters: []byte{0xd1, 0x81}}, + "sdot": {Name: "sdot", CodePoints: []int{8901}, Characters: []byte{0xe2, 0x8b, 0x85}}, + "sdotb": {Name: "sdotb", CodePoints: []int{8865}, Characters: []byte{0xe2, 0x8a, 0xa1}}, + "sdote": {Name: "sdote", CodePoints: []int{10854}, Characters: []byte{0xe2, 0xa9, 0xa6}}, + "seArr": {Name: "seArr", CodePoints: []int{8664}, Characters: []byte{0xe2, 0x87, 0x98}}, + "searhk": {Name: "searhk", CodePoints: []int{10533}, Characters: []byte{0xe2, 0xa4, 0xa5}}, + "searr": {Name: "searr", CodePoints: []int{8600}, Characters: []byte{0xe2, 0x86, 0x98}}, + "searrow": {Name: "searrow", CodePoints: []int{8600}, Characters: []byte{0xe2, 0x86, 0x98}}, + "sect": {Name: "sect", CodePoints: []int{167}, Characters: []byte{0xc2, 0xa7}}, + "semi": {Name: "semi", CodePoints: []int{59}, Characters: []byte{0x3b}}, + "seswar": {Name: "seswar", CodePoints: []int{10537}, Characters: []byte{0xe2, 0xa4, 0xa9}}, + "setminus": {Name: "setminus", CodePoints: []int{8726}, Characters: []byte{0xe2, 0x88, 0x96}}, + "setmn": {Name: "setmn", CodePoints: []int{8726}, Characters: []byte{0xe2, 0x88, 0x96}}, + "sext": {Name: "sext", CodePoints: []int{10038}, Characters: []byte{0xe2, 0x9c, 0xb6}}, + "sfr": {Name: "sfr", CodePoints: []int{120112}, Characters: []byte{0xf0, 0x9d, 0x94, 0xb0}}, + "sfrown": {Name: "sfrown", CodePoints: []int{8994}, Characters: []byte{0xe2, 0x8c, 0xa2}}, + "sharp": {Name: "sharp", CodePoints: []int{9839}, Characters: []byte{0xe2, 0x99, 0xaf}}, + "shchcy": {Name: "shchcy", CodePoints: []int{1097}, Characters: []byte{0xd1, 0x89}}, + "shcy": {Name: "shcy", CodePoints: []int{1096}, Characters: []byte{0xd1, 0x88}}, + "shortmid": {Name: "shortmid", CodePoints: []int{8739}, Characters: []byte{0xe2, 0x88, 0xa3}}, + "shortparallel": {Name: "shortparallel", CodePoints: []int{8741}, Characters: []byte{0xe2, 0x88, 0xa5}}, + "shy": {Name: "shy", CodePoints: []int{173}, Characters: []byte{0xc2, 0xad}}, + "sigma": {Name: "sigma", CodePoints: []int{963}, Characters: []byte{0xcf, 0x83}}, + "sigmaf": {Name: "sigmaf", CodePoints: []int{962}, Characters: []byte{0xcf, 0x82}}, + "sigmav": {Name: "sigmav", CodePoints: []int{962}, Characters: []byte{0xcf, 0x82}}, + "sim": {Name: "sim", CodePoints: []int{8764}, Characters: []byte{0xe2, 0x88, 0xbc}}, + "simdot": {Name: "simdot", CodePoints: []int{10858}, Characters: []byte{0xe2, 0xa9, 0xaa}}, + "sime": {Name: "sime", CodePoints: []int{8771}, Characters: []byte{0xe2, 0x89, 0x83}}, + "simeq": {Name: "simeq", CodePoints: []int{8771}, Characters: []byte{0xe2, 0x89, 0x83}}, + "simg": {Name: "simg", CodePoints: []int{10910}, Characters: []byte{0xe2, 0xaa, 0x9e}}, + "simgE": {Name: "simgE", CodePoints: []int{10912}, Characters: []byte{0xe2, 0xaa, 0xa0}}, + "siml": {Name: "siml", CodePoints: []int{10909}, Characters: []byte{0xe2, 0xaa, 0x9d}}, + "simlE": {Name: "simlE", CodePoints: []int{10911}, Characters: []byte{0xe2, 0xaa, 0x9f}}, + "simne": {Name: "simne", CodePoints: []int{8774}, Characters: []byte{0xe2, 0x89, 0x86}}, + "simplus": {Name: "simplus", CodePoints: []int{10788}, Characters: []byte{0xe2, 0xa8, 0xa4}}, + "simrarr": {Name: "simrarr", CodePoints: []int{10610}, Characters: []byte{0xe2, 0xa5, 0xb2}}, + "slarr": {Name: "slarr", CodePoints: []int{8592}, Characters: []byte{0xe2, 0x86, 0x90}}, + "smallsetminus": {Name: "smallsetminus", CodePoints: []int{8726}, Characters: []byte{0xe2, 0x88, 0x96}}, + "smashp": {Name: "smashp", CodePoints: []int{10803}, Characters: []byte{0xe2, 0xa8, 0xb3}}, + "smeparsl": {Name: "smeparsl", CodePoints: []int{10724}, Characters: []byte{0xe2, 0xa7, 0xa4}}, + "smid": {Name: "smid", CodePoints: []int{8739}, Characters: []byte{0xe2, 0x88, 0xa3}}, + "smile": {Name: "smile", CodePoints: []int{8995}, Characters: []byte{0xe2, 0x8c, 0xa3}}, + "smt": {Name: "smt", CodePoints: []int{10922}, Characters: []byte{0xe2, 0xaa, 0xaa}}, + "smte": {Name: "smte", CodePoints: []int{10924}, Characters: []byte{0xe2, 0xaa, 0xac}}, + "smtes": {Name: "smtes", CodePoints: []int{10924, 65024}, Characters: []byte{0xe2, 0xaa, 0xac, 0xef, 0xb8, 0x80}}, + "softcy": {Name: "softcy", CodePoints: []int{1100}, Characters: []byte{0xd1, 0x8c}}, + "sol": {Name: "sol", CodePoints: []int{47}, Characters: []byte{0x2f}}, + "solb": {Name: "solb", CodePoints: []int{10692}, Characters: []byte{0xe2, 0xa7, 0x84}}, + "solbar": {Name: "solbar", CodePoints: []int{9023}, Characters: []byte{0xe2, 0x8c, 0xbf}}, + "sopf": {Name: "sopf", CodePoints: []int{120164}, Characters: []byte{0xf0, 0x9d, 0x95, 0xa4}}, + "spades": {Name: "spades", CodePoints: []int{9824}, Characters: []byte{0xe2, 0x99, 0xa0}}, + "spadesuit": {Name: "spadesuit", CodePoints: []int{9824}, Characters: []byte{0xe2, 0x99, 0xa0}}, + "spar": {Name: "spar", CodePoints: []int{8741}, Characters: []byte{0xe2, 0x88, 0xa5}}, + "sqcap": {Name: "sqcap", CodePoints: []int{8851}, Characters: []byte{0xe2, 0x8a, 0x93}}, + "sqcaps": {Name: "sqcaps", CodePoints: []int{8851, 65024}, Characters: []byte{0xe2, 0x8a, 0x93, 0xef, 0xb8, 0x80}}, + "sqcup": {Name: "sqcup", CodePoints: []int{8852}, Characters: []byte{0xe2, 0x8a, 0x94}}, + "sqcups": {Name: "sqcups", CodePoints: []int{8852, 65024}, Characters: []byte{0xe2, 0x8a, 0x94, 0xef, 0xb8, 0x80}}, + "sqsub": {Name: "sqsub", CodePoints: []int{8847}, Characters: []byte{0xe2, 0x8a, 0x8f}}, + "sqsube": {Name: "sqsube", CodePoints: []int{8849}, Characters: []byte{0xe2, 0x8a, 0x91}}, + "sqsubset": {Name: "sqsubset", CodePoints: []int{8847}, Characters: []byte{0xe2, 0x8a, 0x8f}}, + "sqsubseteq": {Name: "sqsubseteq", CodePoints: []int{8849}, Characters: []byte{0xe2, 0x8a, 0x91}}, + "sqsup": {Name: "sqsup", CodePoints: []int{8848}, Characters: []byte{0xe2, 0x8a, 0x90}}, + "sqsupe": {Name: "sqsupe", CodePoints: []int{8850}, Characters: []byte{0xe2, 0x8a, 0x92}}, + "sqsupset": {Name: "sqsupset", CodePoints: []int{8848}, Characters: []byte{0xe2, 0x8a, 0x90}}, + "sqsupseteq": {Name: "sqsupseteq", CodePoints: []int{8850}, Characters: []byte{0xe2, 0x8a, 0x92}}, + "squ": {Name: "squ", CodePoints: []int{9633}, Characters: []byte{0xe2, 0x96, 0xa1}}, + "square": {Name: "square", CodePoints: []int{9633}, Characters: []byte{0xe2, 0x96, 0xa1}}, + "squarf": {Name: "squarf", CodePoints: []int{9642}, Characters: []byte{0xe2, 0x96, 0xaa}}, + "squf": {Name: "squf", CodePoints: []int{9642}, Characters: []byte{0xe2, 0x96, 0xaa}}, + "srarr": {Name: "srarr", CodePoints: []int{8594}, Characters: []byte{0xe2, 0x86, 0x92}}, + "sscr": {Name: "sscr", CodePoints: []int{120008}, Characters: []byte{0xf0, 0x9d, 0x93, 0x88}}, + "ssetmn": {Name: "ssetmn", CodePoints: []int{8726}, Characters: []byte{0xe2, 0x88, 0x96}}, + "ssmile": {Name: "ssmile", CodePoints: []int{8995}, Characters: []byte{0xe2, 0x8c, 0xa3}}, + "sstarf": {Name: "sstarf", CodePoints: []int{8902}, Characters: []byte{0xe2, 0x8b, 0x86}}, + "star": {Name: "star", CodePoints: []int{9734}, Characters: []byte{0xe2, 0x98, 0x86}}, + "starf": {Name: "starf", CodePoints: []int{9733}, Characters: []byte{0xe2, 0x98, 0x85}}, + "straightepsilon": {Name: "straightepsilon", CodePoints: []int{1013}, Characters: []byte{0xcf, 0xb5}}, + "straightphi": {Name: "straightphi", CodePoints: []int{981}, Characters: []byte{0xcf, 0x95}}, + "strns": {Name: "strns", CodePoints: []int{175}, Characters: []byte{0xc2, 0xaf}}, + "sub": {Name: "sub", CodePoints: []int{8834}, Characters: []byte{0xe2, 0x8a, 0x82}}, + "subE": {Name: "subE", CodePoints: []int{10949}, Characters: []byte{0xe2, 0xab, 0x85}}, + "subdot": {Name: "subdot", CodePoints: []int{10941}, Characters: []byte{0xe2, 0xaa, 0xbd}}, + "sube": {Name: "sube", CodePoints: []int{8838}, Characters: []byte{0xe2, 0x8a, 0x86}}, + "subedot": {Name: "subedot", CodePoints: []int{10947}, Characters: []byte{0xe2, 0xab, 0x83}}, + "submult": {Name: "submult", CodePoints: []int{10945}, Characters: []byte{0xe2, 0xab, 0x81}}, + "subnE": {Name: "subnE", CodePoints: []int{10955}, Characters: []byte{0xe2, 0xab, 0x8b}}, + "subne": {Name: "subne", CodePoints: []int{8842}, Characters: []byte{0xe2, 0x8a, 0x8a}}, + "subplus": {Name: "subplus", CodePoints: []int{10943}, Characters: []byte{0xe2, 0xaa, 0xbf}}, + "subrarr": {Name: "subrarr", CodePoints: []int{10617}, Characters: []byte{0xe2, 0xa5, 0xb9}}, + "subset": {Name: "subset", CodePoints: []int{8834}, Characters: []byte{0xe2, 0x8a, 0x82}}, + "subseteq": {Name: "subseteq", CodePoints: []int{8838}, Characters: []byte{0xe2, 0x8a, 0x86}}, + "subseteqq": {Name: "subseteqq", CodePoints: []int{10949}, Characters: []byte{0xe2, 0xab, 0x85}}, + "subsetneq": {Name: "subsetneq", CodePoints: []int{8842}, Characters: []byte{0xe2, 0x8a, 0x8a}}, + "subsetneqq": {Name: "subsetneqq", CodePoints: []int{10955}, Characters: []byte{0xe2, 0xab, 0x8b}}, + "subsim": {Name: "subsim", CodePoints: []int{10951}, Characters: []byte{0xe2, 0xab, 0x87}}, + "subsub": {Name: "subsub", CodePoints: []int{10965}, Characters: []byte{0xe2, 0xab, 0x95}}, + "subsup": {Name: "subsup", CodePoints: []int{10963}, Characters: []byte{0xe2, 0xab, 0x93}}, + "succ": {Name: "succ", CodePoints: []int{8827}, Characters: []byte{0xe2, 0x89, 0xbb}}, + "succapprox": {Name: "succapprox", CodePoints: []int{10936}, Characters: []byte{0xe2, 0xaa, 0xb8}}, + "succcurlyeq": {Name: "succcurlyeq", CodePoints: []int{8829}, Characters: []byte{0xe2, 0x89, 0xbd}}, + "succeq": {Name: "succeq", CodePoints: []int{10928}, Characters: []byte{0xe2, 0xaa, 0xb0}}, + "succnapprox": {Name: "succnapprox", CodePoints: []int{10938}, Characters: []byte{0xe2, 0xaa, 0xba}}, + "succneqq": {Name: "succneqq", CodePoints: []int{10934}, Characters: []byte{0xe2, 0xaa, 0xb6}}, + "succnsim": {Name: "succnsim", CodePoints: []int{8937}, Characters: []byte{0xe2, 0x8b, 0xa9}}, + "succsim": {Name: "succsim", CodePoints: []int{8831}, Characters: []byte{0xe2, 0x89, 0xbf}}, + "sum": {Name: "sum", CodePoints: []int{8721}, Characters: []byte{0xe2, 0x88, 0x91}}, + "sung": {Name: "sung", CodePoints: []int{9834}, Characters: []byte{0xe2, 0x99, 0xaa}}, + "sup": {Name: "sup", CodePoints: []int{8835}, Characters: []byte{0xe2, 0x8a, 0x83}}, + "sup1": {Name: "sup1", CodePoints: []int{185}, Characters: []byte{0xc2, 0xb9}}, + "sup2": {Name: "sup2", CodePoints: []int{178}, Characters: []byte{0xc2, 0xb2}}, + "sup3": {Name: "sup3", CodePoints: []int{179}, Characters: []byte{0xc2, 0xb3}}, + "supE": {Name: "supE", CodePoints: []int{10950}, Characters: []byte{0xe2, 0xab, 0x86}}, + "supdot": {Name: "supdot", CodePoints: []int{10942}, Characters: []byte{0xe2, 0xaa, 0xbe}}, + "supdsub": {Name: "supdsub", CodePoints: []int{10968}, Characters: []byte{0xe2, 0xab, 0x98}}, + "supe": {Name: "supe", CodePoints: []int{8839}, Characters: []byte{0xe2, 0x8a, 0x87}}, + "supedot": {Name: "supedot", CodePoints: []int{10948}, Characters: []byte{0xe2, 0xab, 0x84}}, + "suphsol": {Name: "suphsol", CodePoints: []int{10185}, Characters: []byte{0xe2, 0x9f, 0x89}}, + "suphsub": {Name: "suphsub", CodePoints: []int{10967}, Characters: []byte{0xe2, 0xab, 0x97}}, + "suplarr": {Name: "suplarr", CodePoints: []int{10619}, Characters: []byte{0xe2, 0xa5, 0xbb}}, + "supmult": {Name: "supmult", CodePoints: []int{10946}, Characters: []byte{0xe2, 0xab, 0x82}}, + "supnE": {Name: "supnE", CodePoints: []int{10956}, Characters: []byte{0xe2, 0xab, 0x8c}}, + "supne": {Name: "supne", CodePoints: []int{8843}, Characters: []byte{0xe2, 0x8a, 0x8b}}, + "supplus": {Name: "supplus", CodePoints: []int{10944}, Characters: []byte{0xe2, 0xab, 0x80}}, + "supset": {Name: "supset", CodePoints: []int{8835}, Characters: []byte{0xe2, 0x8a, 0x83}}, + "supseteq": {Name: "supseteq", CodePoints: []int{8839}, Characters: []byte{0xe2, 0x8a, 0x87}}, + "supseteqq": {Name: "supseteqq", CodePoints: []int{10950}, Characters: []byte{0xe2, 0xab, 0x86}}, + "supsetneq": {Name: "supsetneq", CodePoints: []int{8843}, Characters: []byte{0xe2, 0x8a, 0x8b}}, + "supsetneqq": {Name: "supsetneqq", CodePoints: []int{10956}, Characters: []byte{0xe2, 0xab, 0x8c}}, + "supsim": {Name: "supsim", CodePoints: []int{10952}, Characters: []byte{0xe2, 0xab, 0x88}}, + "supsub": {Name: "supsub", CodePoints: []int{10964}, Characters: []byte{0xe2, 0xab, 0x94}}, + "supsup": {Name: "supsup", CodePoints: []int{10966}, Characters: []byte{0xe2, 0xab, 0x96}}, + "swArr": {Name: "swArr", CodePoints: []int{8665}, Characters: []byte{0xe2, 0x87, 0x99}}, + "swarhk": {Name: "swarhk", CodePoints: []int{10534}, Characters: []byte{0xe2, 0xa4, 0xa6}}, + "swarr": {Name: "swarr", CodePoints: []int{8601}, Characters: []byte{0xe2, 0x86, 0x99}}, + "swarrow": {Name: "swarrow", CodePoints: []int{8601}, Characters: []byte{0xe2, 0x86, 0x99}}, + "swnwar": {Name: "swnwar", CodePoints: []int{10538}, Characters: []byte{0xe2, 0xa4, 0xaa}}, + "szlig": {Name: "szlig", CodePoints: []int{223}, Characters: []byte{0xc3, 0x9f}}, + "target": {Name: "target", CodePoints: []int{8982}, Characters: []byte{0xe2, 0x8c, 0x96}}, + "tau": {Name: "tau", CodePoints: []int{964}, Characters: []byte{0xcf, 0x84}}, + "tbrk": {Name: "tbrk", CodePoints: []int{9140}, Characters: []byte{0xe2, 0x8e, 0xb4}}, + "tcaron": {Name: "tcaron", CodePoints: []int{357}, Characters: []byte{0xc5, 0xa5}}, + "tcedil": {Name: "tcedil", CodePoints: []int{355}, Characters: []byte{0xc5, 0xa3}}, + "tcy": {Name: "tcy", CodePoints: []int{1090}, Characters: []byte{0xd1, 0x82}}, + "tdot": {Name: "tdot", CodePoints: []int{8411}, Characters: []byte{0xe2, 0x83, 0x9b}}, + "telrec": {Name: "telrec", CodePoints: []int{8981}, Characters: []byte{0xe2, 0x8c, 0x95}}, + "tfr": {Name: "tfr", CodePoints: []int{120113}, Characters: []byte{0xf0, 0x9d, 0x94, 0xb1}}, + "there4": {Name: "there4", CodePoints: []int{8756}, Characters: []byte{0xe2, 0x88, 0xb4}}, + "therefore": {Name: "therefore", CodePoints: []int{8756}, Characters: []byte{0xe2, 0x88, 0xb4}}, + "theta": {Name: "theta", CodePoints: []int{952}, Characters: []byte{0xce, 0xb8}}, + "thetasym": {Name: "thetasym", CodePoints: []int{977}, Characters: []byte{0xcf, 0x91}}, + "thetav": {Name: "thetav", CodePoints: []int{977}, Characters: []byte{0xcf, 0x91}}, + "thickapprox": {Name: "thickapprox", CodePoints: []int{8776}, Characters: []byte{0xe2, 0x89, 0x88}}, + "thicksim": {Name: "thicksim", CodePoints: []int{8764}, Characters: []byte{0xe2, 0x88, 0xbc}}, + "thinsp": {Name: "thinsp", CodePoints: []int{8201}, Characters: []byte{0xe2, 0x80, 0x89}}, + "thkap": {Name: "thkap", CodePoints: []int{8776}, Characters: []byte{0xe2, 0x89, 0x88}}, + "thksim": {Name: "thksim", CodePoints: []int{8764}, Characters: []byte{0xe2, 0x88, 0xbc}}, + "thorn": {Name: "thorn", CodePoints: []int{254}, Characters: []byte{0xc3, 0xbe}}, + "tilde": {Name: "tilde", CodePoints: []int{732}, Characters: []byte{0xcb, 0x9c}}, + "times": {Name: "times", CodePoints: []int{215}, Characters: []byte{0xc3, 0x97}}, + "timesb": {Name: "timesb", CodePoints: []int{8864}, Characters: []byte{0xe2, 0x8a, 0xa0}}, + "timesbar": {Name: "timesbar", CodePoints: []int{10801}, Characters: []byte{0xe2, 0xa8, 0xb1}}, + "timesd": {Name: "timesd", CodePoints: []int{10800}, Characters: []byte{0xe2, 0xa8, 0xb0}}, + "tint": {Name: "tint", CodePoints: []int{8749}, Characters: []byte{0xe2, 0x88, 0xad}}, + "toea": {Name: "toea", CodePoints: []int{10536}, Characters: []byte{0xe2, 0xa4, 0xa8}}, + "top": {Name: "top", CodePoints: []int{8868}, Characters: []byte{0xe2, 0x8a, 0xa4}}, + "topbot": {Name: "topbot", CodePoints: []int{9014}, Characters: []byte{0xe2, 0x8c, 0xb6}}, + "topcir": {Name: "topcir", CodePoints: []int{10993}, Characters: []byte{0xe2, 0xab, 0xb1}}, + "topf": {Name: "topf", CodePoints: []int{120165}, Characters: []byte{0xf0, 0x9d, 0x95, 0xa5}}, + "topfork": {Name: "topfork", CodePoints: []int{10970}, Characters: []byte{0xe2, 0xab, 0x9a}}, + "tosa": {Name: "tosa", CodePoints: []int{10537}, Characters: []byte{0xe2, 0xa4, 0xa9}}, + "tprime": {Name: "tprime", CodePoints: []int{8244}, Characters: []byte{0xe2, 0x80, 0xb4}}, + "trade": {Name: "trade", CodePoints: []int{8482}, Characters: []byte{0xe2, 0x84, 0xa2}}, + "triangle": {Name: "triangle", CodePoints: []int{9653}, Characters: []byte{0xe2, 0x96, 0xb5}}, + "triangledown": {Name: "triangledown", CodePoints: []int{9663}, Characters: []byte{0xe2, 0x96, 0xbf}}, + "triangleleft": {Name: "triangleleft", CodePoints: []int{9667}, Characters: []byte{0xe2, 0x97, 0x83}}, + "trianglelefteq": {Name: "trianglelefteq", CodePoints: []int{8884}, Characters: []byte{0xe2, 0x8a, 0xb4}}, + "triangleq": {Name: "triangleq", CodePoints: []int{8796}, Characters: []byte{0xe2, 0x89, 0x9c}}, + "triangleright": {Name: "triangleright", CodePoints: []int{9657}, Characters: []byte{0xe2, 0x96, 0xb9}}, + "trianglerighteq": {Name: "trianglerighteq", CodePoints: []int{8885}, Characters: []byte{0xe2, 0x8a, 0xb5}}, + "tridot": {Name: "tridot", CodePoints: []int{9708}, Characters: []byte{0xe2, 0x97, 0xac}}, + "trie": {Name: "trie", CodePoints: []int{8796}, Characters: []byte{0xe2, 0x89, 0x9c}}, + "triminus": {Name: "triminus", CodePoints: []int{10810}, Characters: []byte{0xe2, 0xa8, 0xba}}, + "triplus": {Name: "triplus", CodePoints: []int{10809}, Characters: []byte{0xe2, 0xa8, 0xb9}}, + "trisb": {Name: "trisb", CodePoints: []int{10701}, Characters: []byte{0xe2, 0xa7, 0x8d}}, + "tritime": {Name: "tritime", CodePoints: []int{10811}, Characters: []byte{0xe2, 0xa8, 0xbb}}, + "trpezium": {Name: "trpezium", CodePoints: []int{9186}, Characters: []byte{0xe2, 0x8f, 0xa2}}, + "tscr": {Name: "tscr", CodePoints: []int{120009}, Characters: []byte{0xf0, 0x9d, 0x93, 0x89}}, + "tscy": {Name: "tscy", CodePoints: []int{1094}, Characters: []byte{0xd1, 0x86}}, + "tshcy": {Name: "tshcy", CodePoints: []int{1115}, Characters: []byte{0xd1, 0x9b}}, + "tstrok": {Name: "tstrok", CodePoints: []int{359}, Characters: []byte{0xc5, 0xa7}}, + "twixt": {Name: "twixt", CodePoints: []int{8812}, Characters: []byte{0xe2, 0x89, 0xac}}, + "twoheadleftarrow": {Name: "twoheadleftarrow", CodePoints: []int{8606}, Characters: []byte{0xe2, 0x86, 0x9e}}, + "twoheadrightarrow": {Name: "twoheadrightarrow", CodePoints: []int{8608}, Characters: []byte{0xe2, 0x86, 0xa0}}, + "uArr": {Name: "uArr", CodePoints: []int{8657}, Characters: []byte{0xe2, 0x87, 0x91}}, + "uHar": {Name: "uHar", CodePoints: []int{10595}, Characters: []byte{0xe2, 0xa5, 0xa3}}, + "uacute": {Name: "uacute", CodePoints: []int{250}, Characters: []byte{0xc3, 0xba}}, + "uarr": {Name: "uarr", CodePoints: []int{8593}, Characters: []byte{0xe2, 0x86, 0x91}}, + "ubrcy": {Name: "ubrcy", CodePoints: []int{1118}, Characters: []byte{0xd1, 0x9e}}, + "ubreve": {Name: "ubreve", CodePoints: []int{365}, Characters: []byte{0xc5, 0xad}}, + "ucirc": {Name: "ucirc", CodePoints: []int{251}, Characters: []byte{0xc3, 0xbb}}, + "ucy": {Name: "ucy", CodePoints: []int{1091}, Characters: []byte{0xd1, 0x83}}, + "udarr": {Name: "udarr", CodePoints: []int{8645}, Characters: []byte{0xe2, 0x87, 0x85}}, + "udblac": {Name: "udblac", CodePoints: []int{369}, Characters: []byte{0xc5, 0xb1}}, + "udhar": {Name: "udhar", CodePoints: []int{10606}, Characters: []byte{0xe2, 0xa5, 0xae}}, + "ufisht": {Name: "ufisht", CodePoints: []int{10622}, Characters: []byte{0xe2, 0xa5, 0xbe}}, + "ufr": {Name: "ufr", CodePoints: []int{120114}, Characters: []byte{0xf0, 0x9d, 0x94, 0xb2}}, + "ugrave": {Name: "ugrave", CodePoints: []int{249}, Characters: []byte{0xc3, 0xb9}}, + "uharl": {Name: "uharl", CodePoints: []int{8639}, Characters: []byte{0xe2, 0x86, 0xbf}}, + "uharr": {Name: "uharr", CodePoints: []int{8638}, Characters: []byte{0xe2, 0x86, 0xbe}}, + "uhblk": {Name: "uhblk", CodePoints: []int{9600}, Characters: []byte{0xe2, 0x96, 0x80}}, + "ulcorn": {Name: "ulcorn", CodePoints: []int{8988}, Characters: []byte{0xe2, 0x8c, 0x9c}}, + "ulcorner": {Name: "ulcorner", CodePoints: []int{8988}, Characters: []byte{0xe2, 0x8c, 0x9c}}, + "ulcrop": {Name: "ulcrop", CodePoints: []int{8975}, Characters: []byte{0xe2, 0x8c, 0x8f}}, + "ultri": {Name: "ultri", CodePoints: []int{9720}, Characters: []byte{0xe2, 0x97, 0xb8}}, + "umacr": {Name: "umacr", CodePoints: []int{363}, Characters: []byte{0xc5, 0xab}}, + "uml": {Name: "uml", CodePoints: []int{168}, Characters: []byte{0xc2, 0xa8}}, + "uogon": {Name: "uogon", CodePoints: []int{371}, Characters: []byte{0xc5, 0xb3}}, + "uopf": {Name: "uopf", CodePoints: []int{120166}, Characters: []byte{0xf0, 0x9d, 0x95, 0xa6}}, + "uparrow": {Name: "uparrow", CodePoints: []int{8593}, Characters: []byte{0xe2, 0x86, 0x91}}, + "updownarrow": {Name: "updownarrow", CodePoints: []int{8597}, Characters: []byte{0xe2, 0x86, 0x95}}, + "upharpoonleft": {Name: "upharpoonleft", CodePoints: []int{8639}, Characters: []byte{0xe2, 0x86, 0xbf}}, + "upharpoonright": {Name: "upharpoonright", CodePoints: []int{8638}, Characters: []byte{0xe2, 0x86, 0xbe}}, + "uplus": {Name: "uplus", CodePoints: []int{8846}, Characters: []byte{0xe2, 0x8a, 0x8e}}, + "upsi": {Name: "upsi", CodePoints: []int{965}, Characters: []byte{0xcf, 0x85}}, + "upsih": {Name: "upsih", CodePoints: []int{978}, Characters: []byte{0xcf, 0x92}}, + "upsilon": {Name: "upsilon", CodePoints: []int{965}, Characters: []byte{0xcf, 0x85}}, + "upuparrows": {Name: "upuparrows", CodePoints: []int{8648}, Characters: []byte{0xe2, 0x87, 0x88}}, + "urcorn": {Name: "urcorn", CodePoints: []int{8989}, Characters: []byte{0xe2, 0x8c, 0x9d}}, + "urcorner": {Name: "urcorner", CodePoints: []int{8989}, Characters: []byte{0xe2, 0x8c, 0x9d}}, + "urcrop": {Name: "urcrop", CodePoints: []int{8974}, Characters: []byte{0xe2, 0x8c, 0x8e}}, + "uring": {Name: "uring", CodePoints: []int{367}, Characters: []byte{0xc5, 0xaf}}, + "urtri": {Name: "urtri", CodePoints: []int{9721}, Characters: []byte{0xe2, 0x97, 0xb9}}, + "uscr": {Name: "uscr", CodePoints: []int{120010}, Characters: []byte{0xf0, 0x9d, 0x93, 0x8a}}, + "utdot": {Name: "utdot", CodePoints: []int{8944}, Characters: []byte{0xe2, 0x8b, 0xb0}}, + "utilde": {Name: "utilde", CodePoints: []int{361}, Characters: []byte{0xc5, 0xa9}}, + "utri": {Name: "utri", CodePoints: []int{9653}, Characters: []byte{0xe2, 0x96, 0xb5}}, + "utrif": {Name: "utrif", CodePoints: []int{9652}, Characters: []byte{0xe2, 0x96, 0xb4}}, + "uuarr": {Name: "uuarr", CodePoints: []int{8648}, Characters: []byte{0xe2, 0x87, 0x88}}, + "uuml": {Name: "uuml", CodePoints: []int{252}, Characters: []byte{0xc3, 0xbc}}, + "uwangle": {Name: "uwangle", CodePoints: []int{10663}, Characters: []byte{0xe2, 0xa6, 0xa7}}, + "vArr": {Name: "vArr", CodePoints: []int{8661}, Characters: []byte{0xe2, 0x87, 0x95}}, + "vBar": {Name: "vBar", CodePoints: []int{10984}, Characters: []byte{0xe2, 0xab, 0xa8}}, + "vBarv": {Name: "vBarv", CodePoints: []int{10985}, Characters: []byte{0xe2, 0xab, 0xa9}}, + "vDash": {Name: "vDash", CodePoints: []int{8872}, Characters: []byte{0xe2, 0x8a, 0xa8}}, + "vangrt": {Name: "vangrt", CodePoints: []int{10652}, Characters: []byte{0xe2, 0xa6, 0x9c}}, + "varepsilon": {Name: "varepsilon", CodePoints: []int{1013}, Characters: []byte{0xcf, 0xb5}}, + "varkappa": {Name: "varkappa", CodePoints: []int{1008}, Characters: []byte{0xcf, 0xb0}}, + "varnothing": {Name: "varnothing", CodePoints: []int{8709}, Characters: []byte{0xe2, 0x88, 0x85}}, + "varphi": {Name: "varphi", CodePoints: []int{981}, Characters: []byte{0xcf, 0x95}}, + "varpi": {Name: "varpi", CodePoints: []int{982}, Characters: []byte{0xcf, 0x96}}, + "varpropto": {Name: "varpropto", CodePoints: []int{8733}, Characters: []byte{0xe2, 0x88, 0x9d}}, + "varr": {Name: "varr", CodePoints: []int{8597}, Characters: []byte{0xe2, 0x86, 0x95}}, + "varrho": {Name: "varrho", CodePoints: []int{1009}, Characters: []byte{0xcf, 0xb1}}, + "varsigma": {Name: "varsigma", CodePoints: []int{962}, Characters: []byte{0xcf, 0x82}}, + "varsubsetneq": {Name: "varsubsetneq", CodePoints: []int{8842, 65024}, Characters: []byte{0xe2, 0x8a, 0x8a, 0xef, 0xb8, 0x80}}, + "varsubsetneqq": {Name: "varsubsetneqq", CodePoints: []int{10955, 65024}, Characters: []byte{0xe2, 0xab, 0x8b, 0xef, 0xb8, 0x80}}, + "varsupsetneq": {Name: "varsupsetneq", CodePoints: []int{8843, 65024}, Characters: []byte{0xe2, 0x8a, 0x8b, 0xef, 0xb8, 0x80}}, + "varsupsetneqq": {Name: "varsupsetneqq", CodePoints: []int{10956, 65024}, Characters: []byte{0xe2, 0xab, 0x8c, 0xef, 0xb8, 0x80}}, + "vartheta": {Name: "vartheta", CodePoints: []int{977}, Characters: []byte{0xcf, 0x91}}, + "vartriangleleft": {Name: "vartriangleleft", CodePoints: []int{8882}, Characters: []byte{0xe2, 0x8a, 0xb2}}, + "vartriangleright": {Name: "vartriangleright", CodePoints: []int{8883}, Characters: []byte{0xe2, 0x8a, 0xb3}}, + "vcy": {Name: "vcy", CodePoints: []int{1074}, Characters: []byte{0xd0, 0xb2}}, + "vdash": {Name: "vdash", CodePoints: []int{8866}, Characters: []byte{0xe2, 0x8a, 0xa2}}, + "vee": {Name: "vee", CodePoints: []int{8744}, Characters: []byte{0xe2, 0x88, 0xa8}}, + "veebar": {Name: "veebar", CodePoints: []int{8891}, Characters: []byte{0xe2, 0x8a, 0xbb}}, + "veeeq": {Name: "veeeq", CodePoints: []int{8794}, Characters: []byte{0xe2, 0x89, 0x9a}}, + "vellip": {Name: "vellip", CodePoints: []int{8942}, Characters: []byte{0xe2, 0x8b, 0xae}}, + "verbar": {Name: "verbar", CodePoints: []int{124}, Characters: []byte{0x7c}}, + "vert": {Name: "vert", CodePoints: []int{124}, Characters: []byte{0x7c}}, + "vfr": {Name: "vfr", CodePoints: []int{120115}, Characters: []byte{0xf0, 0x9d, 0x94, 0xb3}}, + "vltri": {Name: "vltri", CodePoints: []int{8882}, Characters: []byte{0xe2, 0x8a, 0xb2}}, + "vnsub": {Name: "vnsub", CodePoints: []int{8834, 8402}, Characters: []byte{0xe2, 0x8a, 0x82, 0xe2, 0x83, 0x92}}, + "vnsup": {Name: "vnsup", CodePoints: []int{8835, 8402}, Characters: []byte{0xe2, 0x8a, 0x83, 0xe2, 0x83, 0x92}}, + "vopf": {Name: "vopf", CodePoints: []int{120167}, Characters: []byte{0xf0, 0x9d, 0x95, 0xa7}}, + "vprop": {Name: "vprop", CodePoints: []int{8733}, Characters: []byte{0xe2, 0x88, 0x9d}}, + "vrtri": {Name: "vrtri", CodePoints: []int{8883}, Characters: []byte{0xe2, 0x8a, 0xb3}}, + "vscr": {Name: "vscr", CodePoints: []int{120011}, Characters: []byte{0xf0, 0x9d, 0x93, 0x8b}}, + "vsubnE": {Name: "vsubnE", CodePoints: []int{10955, 65024}, Characters: []byte{0xe2, 0xab, 0x8b, 0xef, 0xb8, 0x80}}, + "vsubne": {Name: "vsubne", CodePoints: []int{8842, 65024}, Characters: []byte{0xe2, 0x8a, 0x8a, 0xef, 0xb8, 0x80}}, + "vsupnE": {Name: "vsupnE", CodePoints: []int{10956, 65024}, Characters: []byte{0xe2, 0xab, 0x8c, 0xef, 0xb8, 0x80}}, + "vsupne": {Name: "vsupne", CodePoints: []int{8843, 65024}, Characters: []byte{0xe2, 0x8a, 0x8b, 0xef, 0xb8, 0x80}}, + "vzigzag": {Name: "vzigzag", CodePoints: []int{10650}, Characters: []byte{0xe2, 0xa6, 0x9a}}, + "wcirc": {Name: "wcirc", CodePoints: []int{373}, Characters: []byte{0xc5, 0xb5}}, + "wedbar": {Name: "wedbar", CodePoints: []int{10847}, Characters: []byte{0xe2, 0xa9, 0x9f}}, + "wedge": {Name: "wedge", CodePoints: []int{8743}, Characters: []byte{0xe2, 0x88, 0xa7}}, + "wedgeq": {Name: "wedgeq", CodePoints: []int{8793}, Characters: []byte{0xe2, 0x89, 0x99}}, + "weierp": {Name: "weierp", CodePoints: []int{8472}, Characters: []byte{0xe2, 0x84, 0x98}}, + "wfr": {Name: "wfr", CodePoints: []int{120116}, Characters: []byte{0xf0, 0x9d, 0x94, 0xb4}}, + "wopf": {Name: "wopf", CodePoints: []int{120168}, Characters: []byte{0xf0, 0x9d, 0x95, 0xa8}}, + "wp": {Name: "wp", CodePoints: []int{8472}, Characters: []byte{0xe2, 0x84, 0x98}}, + "wr": {Name: "wr", CodePoints: []int{8768}, Characters: []byte{0xe2, 0x89, 0x80}}, + "wreath": {Name: "wreath", CodePoints: []int{8768}, Characters: []byte{0xe2, 0x89, 0x80}}, + "wscr": {Name: "wscr", CodePoints: []int{120012}, Characters: []byte{0xf0, 0x9d, 0x93, 0x8c}}, + "xcap": {Name: "xcap", CodePoints: []int{8898}, Characters: []byte{0xe2, 0x8b, 0x82}}, + "xcirc": {Name: "xcirc", CodePoints: []int{9711}, Characters: []byte{0xe2, 0x97, 0xaf}}, + "xcup": {Name: "xcup", CodePoints: []int{8899}, Characters: []byte{0xe2, 0x8b, 0x83}}, + "xdtri": {Name: "xdtri", CodePoints: []int{9661}, Characters: []byte{0xe2, 0x96, 0xbd}}, + "xfr": {Name: "xfr", CodePoints: []int{120117}, Characters: []byte{0xf0, 0x9d, 0x94, 0xb5}}, + "xhArr": {Name: "xhArr", CodePoints: []int{10234}, Characters: []byte{0xe2, 0x9f, 0xba}}, + "xharr": {Name: "xharr", CodePoints: []int{10231}, Characters: []byte{0xe2, 0x9f, 0xb7}}, + "xi": {Name: "xi", CodePoints: []int{958}, Characters: []byte{0xce, 0xbe}}, + "xlArr": {Name: "xlArr", CodePoints: []int{10232}, Characters: []byte{0xe2, 0x9f, 0xb8}}, + "xlarr": {Name: "xlarr", CodePoints: []int{10229}, Characters: []byte{0xe2, 0x9f, 0xb5}}, + "xmap": {Name: "xmap", CodePoints: []int{10236}, Characters: []byte{0xe2, 0x9f, 0xbc}}, + "xnis": {Name: "xnis", CodePoints: []int{8955}, Characters: []byte{0xe2, 0x8b, 0xbb}}, + "xodot": {Name: "xodot", CodePoints: []int{10752}, Characters: []byte{0xe2, 0xa8, 0x80}}, + "xopf": {Name: "xopf", CodePoints: []int{120169}, Characters: []byte{0xf0, 0x9d, 0x95, 0xa9}}, + "xoplus": {Name: "xoplus", CodePoints: []int{10753}, Characters: []byte{0xe2, 0xa8, 0x81}}, + "xotime": {Name: "xotime", CodePoints: []int{10754}, Characters: []byte{0xe2, 0xa8, 0x82}}, + "xrArr": {Name: "xrArr", CodePoints: []int{10233}, Characters: []byte{0xe2, 0x9f, 0xb9}}, + "xrarr": {Name: "xrarr", CodePoints: []int{10230}, Characters: []byte{0xe2, 0x9f, 0xb6}}, + "xscr": {Name: "xscr", CodePoints: []int{120013}, Characters: []byte{0xf0, 0x9d, 0x93, 0x8d}}, + "xsqcup": {Name: "xsqcup", CodePoints: []int{10758}, Characters: []byte{0xe2, 0xa8, 0x86}}, + "xuplus": {Name: "xuplus", CodePoints: []int{10756}, Characters: []byte{0xe2, 0xa8, 0x84}}, + "xutri": {Name: "xutri", CodePoints: []int{9651}, Characters: []byte{0xe2, 0x96, 0xb3}}, + "xvee": {Name: "xvee", CodePoints: []int{8897}, Characters: []byte{0xe2, 0x8b, 0x81}}, + "xwedge": {Name: "xwedge", CodePoints: []int{8896}, Characters: []byte{0xe2, 0x8b, 0x80}}, + "yacute": {Name: "yacute", CodePoints: []int{253}, Characters: []byte{0xc3, 0xbd}}, + "yacy": {Name: "yacy", CodePoints: []int{1103}, Characters: []byte{0xd1, 0x8f}}, + "ycirc": {Name: "ycirc", CodePoints: []int{375}, Characters: []byte{0xc5, 0xb7}}, + "ycy": {Name: "ycy", CodePoints: []int{1099}, Characters: []byte{0xd1, 0x8b}}, + "yen": {Name: "yen", CodePoints: []int{165}, Characters: []byte{0xc2, 0xa5}}, + "yfr": {Name: "yfr", CodePoints: []int{120118}, Characters: []byte{0xf0, 0x9d, 0x94, 0xb6}}, + "yicy": {Name: "yicy", CodePoints: []int{1111}, Characters: []byte{0xd1, 0x97}}, + "yopf": {Name: "yopf", CodePoints: []int{120170}, Characters: []byte{0xf0, 0x9d, 0x95, 0xaa}}, + "yscr": {Name: "yscr", CodePoints: []int{120014}, Characters: []byte{0xf0, 0x9d, 0x93, 0x8e}}, + "yucy": {Name: "yucy", CodePoints: []int{1102}, Characters: []byte{0xd1, 0x8e}}, + "yuml": {Name: "yuml", CodePoints: []int{255}, Characters: []byte{0xc3, 0xbf}}, + "zacute": {Name: "zacute", CodePoints: []int{378}, Characters: []byte{0xc5, 0xba}}, + "zcaron": {Name: "zcaron", CodePoints: []int{382}, Characters: []byte{0xc5, 0xbe}}, + "zcy": {Name: "zcy", CodePoints: []int{1079}, Characters: []byte{0xd0, 0xb7}}, + "zdot": {Name: "zdot", CodePoints: []int{380}, Characters: []byte{0xc5, 0xbc}}, + "zeetrf": {Name: "zeetrf", CodePoints: []int{8488}, Characters: []byte{0xe2, 0x84, 0xa8}}, + "zeta": {Name: "zeta", CodePoints: []int{950}, Characters: []byte{0xce, 0xb6}}, + "zfr": {Name: "zfr", CodePoints: []int{120119}, Characters: []byte{0xf0, 0x9d, 0x94, 0xb7}}, + "zhcy": {Name: "zhcy", CodePoints: []int{1078}, Characters: []byte{0xd0, 0xb6}}, + "zigrarr": {Name: "zigrarr", CodePoints: []int{8669}, Characters: []byte{0xe2, 0x87, 0x9d}}, + "zopf": {Name: "zopf", CodePoints: []int{120171}, Characters: []byte{0xf0, 0x9d, 0x95, 0xab}}, + "zscr": {Name: "zscr", CodePoints: []int{120015}, Characters: []byte{0xf0, 0x9d, 0x93, 0x8f}}, + "zwj": {Name: "zwj", CodePoints: []int{8205}, Characters: []byte{0xe2, 0x80, 0x8d}}, + "zwnj": {Name: "zwnj", CodePoints: []int{8204}, Characters: []byte{0xe2, 0x80, 0x8c}}, +} diff --git a/vendor/github.com/yuin/goldmark/util/unicode_case_folding.go b/vendor/github.com/yuin/goldmark/util/unicode_case_folding.go new file mode 100644 index 00000000..f66ee7c4 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/util/unicode_case_folding.go @@ -0,0 +1,1491 @@ +package util + +var unicodeCaseFoldings = map[rune][]rune{ + 0x41: []int32{97}, + 0x42: []int32{98}, + 0x43: []int32{99}, + 0x44: []int32{100}, + 0x45: []int32{101}, + 0x46: []int32{102}, + 0x47: []int32{103}, + 0x48: []int32{104}, + 0x49: []int32{105}, + 0x4a: []int32{106}, + 0x4b: []int32{107}, + 0x4c: []int32{108}, + 0x4d: []int32{109}, + 0x4e: []int32{110}, + 0x4f: []int32{111}, + 0x50: []int32{112}, + 0x51: []int32{113}, + 0x52: []int32{114}, + 0x53: []int32{115}, + 0x54: []int32{116}, + 0x55: []int32{117}, + 0x56: []int32{118}, + 0x57: []int32{119}, + 0x58: []int32{120}, + 0x59: []int32{121}, + 0x5a: []int32{122}, + 0xb5: []int32{956}, + 0xc0: []int32{224}, + 0xc1: []int32{225}, + 0xc2: []int32{226}, + 0xc3: []int32{227}, + 0xc4: []int32{228}, + 0xc5: []int32{229}, + 0xc6: []int32{230}, + 0xc7: []int32{231}, + 0xc8: []int32{232}, + 0xc9: []int32{233}, + 0xca: []int32{234}, + 0xcb: []int32{235}, + 0xcc: []int32{236}, + 0xcd: []int32{237}, + 0xce: []int32{238}, + 0xcf: []int32{239}, + 0xd0: []int32{240}, + 0xd1: []int32{241}, + 0xd2: []int32{242}, + 0xd3: []int32{243}, + 0xd4: []int32{244}, + 0xd5: []int32{245}, + 0xd6: []int32{246}, + 0xd8: []int32{248}, + 0xd9: []int32{249}, + 0xda: []int32{250}, + 0xdb: []int32{251}, + 0xdc: []int32{252}, + 0xdd: []int32{253}, + 0xde: []int32{254}, + 0xdf: []int32{115, 115}, + 0x100: []int32{257}, + 0x102: []int32{259}, + 0x104: []int32{261}, + 0x106: []int32{263}, + 0x108: []int32{265}, + 0x10a: []int32{267}, + 0x10c: []int32{269}, + 0x10e: []int32{271}, + 0x110: []int32{273}, + 0x112: []int32{275}, + 0x114: []int32{277}, + 0x116: []int32{279}, + 0x118: []int32{281}, + 0x11a: []int32{283}, + 0x11c: []int32{285}, + 0x11e: []int32{287}, + 0x120: []int32{289}, + 0x122: []int32{291}, + 0x124: []int32{293}, + 0x126: []int32{295}, + 0x128: []int32{297}, + 0x12a: []int32{299}, + 0x12c: []int32{301}, + 0x12e: []int32{303}, + 0x130: []int32{105, 775}, + 0x132: []int32{307}, + 0x134: []int32{309}, + 0x136: []int32{311}, + 0x139: []int32{314}, + 0x13b: []int32{316}, + 0x13d: []int32{318}, + 0x13f: []int32{320}, + 0x141: []int32{322}, + 0x143: []int32{324}, + 0x145: []int32{326}, + 0x147: []int32{328}, + 0x149: []int32{700, 110}, + 0x14a: []int32{331}, + 0x14c: []int32{333}, + 0x14e: []int32{335}, + 0x150: []int32{337}, + 0x152: []int32{339}, + 0x154: []int32{341}, + 0x156: []int32{343}, + 0x158: []int32{345}, + 0x15a: []int32{347}, + 0x15c: []int32{349}, + 0x15e: []int32{351}, + 0x160: []int32{353}, + 0x162: []int32{355}, + 0x164: []int32{357}, + 0x166: []int32{359}, + 0x168: []int32{361}, + 0x16a: []int32{363}, + 0x16c: []int32{365}, + 0x16e: []int32{367}, + 0x170: []int32{369}, + 0x172: []int32{371}, + 0x174: []int32{373}, + 0x176: []int32{375}, + 0x178: []int32{255}, + 0x179: []int32{378}, + 0x17b: []int32{380}, + 0x17d: []int32{382}, + 0x17f: []int32{115}, + 0x181: []int32{595}, + 0x182: []int32{387}, + 0x184: []int32{389}, + 0x186: []int32{596}, + 0x187: []int32{392}, + 0x189: []int32{598}, + 0x18a: []int32{599}, + 0x18b: []int32{396}, + 0x18e: []int32{477}, + 0x18f: []int32{601}, + 0x190: []int32{603}, + 0x191: []int32{402}, + 0x193: []int32{608}, + 0x194: []int32{611}, + 0x196: []int32{617}, + 0x197: []int32{616}, + 0x198: []int32{409}, + 0x19c: []int32{623}, + 0x19d: []int32{626}, + 0x19f: []int32{629}, + 0x1a0: []int32{417}, + 0x1a2: []int32{419}, + 0x1a4: []int32{421}, + 0x1a6: []int32{640}, + 0x1a7: []int32{424}, + 0x1a9: []int32{643}, + 0x1ac: []int32{429}, + 0x1ae: []int32{648}, + 0x1af: []int32{432}, + 0x1b1: []int32{650}, + 0x1b2: []int32{651}, + 0x1b3: []int32{436}, + 0x1b5: []int32{438}, + 0x1b7: []int32{658}, + 0x1b8: []int32{441}, + 0x1bc: []int32{445}, + 0x1c4: []int32{454}, + 0x1c5: []int32{454}, + 0x1c7: []int32{457}, + 0x1c8: []int32{457}, + 0x1ca: []int32{460}, + 0x1cb: []int32{460}, + 0x1cd: []int32{462}, + 0x1cf: []int32{464}, + 0x1d1: []int32{466}, + 0x1d3: []int32{468}, + 0x1d5: []int32{470}, + 0x1d7: []int32{472}, + 0x1d9: []int32{474}, + 0x1db: []int32{476}, + 0x1de: []int32{479}, + 0x1e0: []int32{481}, + 0x1e2: []int32{483}, + 0x1e4: []int32{485}, + 0x1e6: []int32{487}, + 0x1e8: []int32{489}, + 0x1ea: []int32{491}, + 0x1ec: []int32{493}, + 0x1ee: []int32{495}, + 0x1f0: []int32{106, 780}, + 0x1f1: []int32{499}, + 0x1f2: []int32{499}, + 0x1f4: []int32{501}, + 0x1f6: []int32{405}, + 0x1f7: []int32{447}, + 0x1f8: []int32{505}, + 0x1fa: []int32{507}, + 0x1fc: []int32{509}, + 0x1fe: []int32{511}, + 0x200: []int32{513}, + 0x202: []int32{515}, + 0x204: []int32{517}, + 0x206: []int32{519}, + 0x208: []int32{521}, + 0x20a: []int32{523}, + 0x20c: []int32{525}, + 0x20e: []int32{527}, + 0x210: []int32{529}, + 0x212: []int32{531}, + 0x214: []int32{533}, + 0x216: []int32{535}, + 0x218: []int32{537}, + 0x21a: []int32{539}, + 0x21c: []int32{541}, + 0x21e: []int32{543}, + 0x220: []int32{414}, + 0x222: []int32{547}, + 0x224: []int32{549}, + 0x226: []int32{551}, + 0x228: []int32{553}, + 0x22a: []int32{555}, + 0x22c: []int32{557}, + 0x22e: []int32{559}, + 0x230: []int32{561}, + 0x232: []int32{563}, + 0x23a: []int32{11365}, + 0x23b: []int32{572}, + 0x23d: []int32{410}, + 0x23e: []int32{11366}, + 0x241: []int32{578}, + 0x243: []int32{384}, + 0x244: []int32{649}, + 0x245: []int32{652}, + 0x246: []int32{583}, + 0x248: []int32{585}, + 0x24a: []int32{587}, + 0x24c: []int32{589}, + 0x24e: []int32{591}, + 0x345: []int32{953}, + 0x370: []int32{881}, + 0x372: []int32{883}, + 0x376: []int32{887}, + 0x37f: []int32{1011}, + 0x386: []int32{940}, + 0x388: []int32{941}, + 0x389: []int32{942}, + 0x38a: []int32{943}, + 0x38c: []int32{972}, + 0x38e: []int32{973}, + 0x38f: []int32{974}, + 0x390: []int32{953, 776, 769}, + 0x391: []int32{945}, + 0x392: []int32{946}, + 0x393: []int32{947}, + 0x394: []int32{948}, + 0x395: []int32{949}, + 0x396: []int32{950}, + 0x397: []int32{951}, + 0x398: []int32{952}, + 0x399: []int32{953}, + 0x39a: []int32{954}, + 0x39b: []int32{955}, + 0x39c: []int32{956}, + 0x39d: []int32{957}, + 0x39e: []int32{958}, + 0x39f: []int32{959}, + 0x3a0: []int32{960}, + 0x3a1: []int32{961}, + 0x3a3: []int32{963}, + 0x3a4: []int32{964}, + 0x3a5: []int32{965}, + 0x3a6: []int32{966}, + 0x3a7: []int32{967}, + 0x3a8: []int32{968}, + 0x3a9: []int32{969}, + 0x3aa: []int32{970}, + 0x3ab: []int32{971}, + 0x3b0: []int32{965, 776, 769}, + 0x3c2: []int32{963}, + 0x3cf: []int32{983}, + 0x3d0: []int32{946}, + 0x3d1: []int32{952}, + 0x3d5: []int32{966}, + 0x3d6: []int32{960}, + 0x3d8: []int32{985}, + 0x3da: []int32{987}, + 0x3dc: []int32{989}, + 0x3de: []int32{991}, + 0x3e0: []int32{993}, + 0x3e2: []int32{995}, + 0x3e4: []int32{997}, + 0x3e6: []int32{999}, + 0x3e8: []int32{1001}, + 0x3ea: []int32{1003}, + 0x3ec: []int32{1005}, + 0x3ee: []int32{1007}, + 0x3f0: []int32{954}, + 0x3f1: []int32{961}, + 0x3f4: []int32{952}, + 0x3f5: []int32{949}, + 0x3f7: []int32{1016}, + 0x3f9: []int32{1010}, + 0x3fa: []int32{1019}, + 0x3fd: []int32{891}, + 0x3fe: []int32{892}, + 0x3ff: []int32{893}, + 0x400: []int32{1104}, + 0x401: []int32{1105}, + 0x402: []int32{1106}, + 0x403: []int32{1107}, + 0x404: []int32{1108}, + 0x405: []int32{1109}, + 0x406: []int32{1110}, + 0x407: []int32{1111}, + 0x408: []int32{1112}, + 0x409: []int32{1113}, + 0x40a: []int32{1114}, + 0x40b: []int32{1115}, + 0x40c: []int32{1116}, + 0x40d: []int32{1117}, + 0x40e: []int32{1118}, + 0x40f: []int32{1119}, + 0x410: []int32{1072}, + 0x411: []int32{1073}, + 0x412: []int32{1074}, + 0x413: []int32{1075}, + 0x414: []int32{1076}, + 0x415: []int32{1077}, + 0x416: []int32{1078}, + 0x417: []int32{1079}, + 0x418: []int32{1080}, + 0x419: []int32{1081}, + 0x41a: []int32{1082}, + 0x41b: []int32{1083}, + 0x41c: []int32{1084}, + 0x41d: []int32{1085}, + 0x41e: []int32{1086}, + 0x41f: []int32{1087}, + 0x420: []int32{1088}, + 0x421: []int32{1089}, + 0x422: []int32{1090}, + 0x423: []int32{1091}, + 0x424: []int32{1092}, + 0x425: []int32{1093}, + 0x426: []int32{1094}, + 0x427: []int32{1095}, + 0x428: []int32{1096}, + 0x429: []int32{1097}, + 0x42a: []int32{1098}, + 0x42b: []int32{1099}, + 0x42c: []int32{1100}, + 0x42d: []int32{1101}, + 0x42e: []int32{1102}, + 0x42f: []int32{1103}, + 0x460: []int32{1121}, + 0x462: []int32{1123}, + 0x464: []int32{1125}, + 0x466: []int32{1127}, + 0x468: []int32{1129}, + 0x46a: []int32{1131}, + 0x46c: []int32{1133}, + 0x46e: []int32{1135}, + 0x470: []int32{1137}, + 0x472: []int32{1139}, + 0x474: []int32{1141}, + 0x476: []int32{1143}, + 0x478: []int32{1145}, + 0x47a: []int32{1147}, + 0x47c: []int32{1149}, + 0x47e: []int32{1151}, + 0x480: []int32{1153}, + 0x48a: []int32{1163}, + 0x48c: []int32{1165}, + 0x48e: []int32{1167}, + 0x490: []int32{1169}, + 0x492: []int32{1171}, + 0x494: []int32{1173}, + 0x496: []int32{1175}, + 0x498: []int32{1177}, + 0x49a: []int32{1179}, + 0x49c: []int32{1181}, + 0x49e: []int32{1183}, + 0x4a0: []int32{1185}, + 0x4a2: []int32{1187}, + 0x4a4: []int32{1189}, + 0x4a6: []int32{1191}, + 0x4a8: []int32{1193}, + 0x4aa: []int32{1195}, + 0x4ac: []int32{1197}, + 0x4ae: []int32{1199}, + 0x4b0: []int32{1201}, + 0x4b2: []int32{1203}, + 0x4b4: []int32{1205}, + 0x4b6: []int32{1207}, + 0x4b8: []int32{1209}, + 0x4ba: []int32{1211}, + 0x4bc: []int32{1213}, + 0x4be: []int32{1215}, + 0x4c0: []int32{1231}, + 0x4c1: []int32{1218}, + 0x4c3: []int32{1220}, + 0x4c5: []int32{1222}, + 0x4c7: []int32{1224}, + 0x4c9: []int32{1226}, + 0x4cb: []int32{1228}, + 0x4cd: []int32{1230}, + 0x4d0: []int32{1233}, + 0x4d2: []int32{1235}, + 0x4d4: []int32{1237}, + 0x4d6: []int32{1239}, + 0x4d8: []int32{1241}, + 0x4da: []int32{1243}, + 0x4dc: []int32{1245}, + 0x4de: []int32{1247}, + 0x4e0: []int32{1249}, + 0x4e2: []int32{1251}, + 0x4e4: []int32{1253}, + 0x4e6: []int32{1255}, + 0x4e8: []int32{1257}, + 0x4ea: []int32{1259}, + 0x4ec: []int32{1261}, + 0x4ee: []int32{1263}, + 0x4f0: []int32{1265}, + 0x4f2: []int32{1267}, + 0x4f4: []int32{1269}, + 0x4f6: []int32{1271}, + 0x4f8: []int32{1273}, + 0x4fa: []int32{1275}, + 0x4fc: []int32{1277}, + 0x4fe: []int32{1279}, + 0x500: []int32{1281}, + 0x502: []int32{1283}, + 0x504: []int32{1285}, + 0x506: []int32{1287}, + 0x508: []int32{1289}, + 0x50a: []int32{1291}, + 0x50c: []int32{1293}, + 0x50e: []int32{1295}, + 0x510: []int32{1297}, + 0x512: []int32{1299}, + 0x514: []int32{1301}, + 0x516: []int32{1303}, + 0x518: []int32{1305}, + 0x51a: []int32{1307}, + 0x51c: []int32{1309}, + 0x51e: []int32{1311}, + 0x520: []int32{1313}, + 0x522: []int32{1315}, + 0x524: []int32{1317}, + 0x526: []int32{1319}, + 0x528: []int32{1321}, + 0x52a: []int32{1323}, + 0x52c: []int32{1325}, + 0x52e: []int32{1327}, + 0x531: []int32{1377}, + 0x532: []int32{1378}, + 0x533: []int32{1379}, + 0x534: []int32{1380}, + 0x535: []int32{1381}, + 0x536: []int32{1382}, + 0x537: []int32{1383}, + 0x538: []int32{1384}, + 0x539: []int32{1385}, + 0x53a: []int32{1386}, + 0x53b: []int32{1387}, + 0x53c: []int32{1388}, + 0x53d: []int32{1389}, + 0x53e: []int32{1390}, + 0x53f: []int32{1391}, + 0x540: []int32{1392}, + 0x541: []int32{1393}, + 0x542: []int32{1394}, + 0x543: []int32{1395}, + 0x544: []int32{1396}, + 0x545: []int32{1397}, + 0x546: []int32{1398}, + 0x547: []int32{1399}, + 0x548: []int32{1400}, + 0x549: []int32{1401}, + 0x54a: []int32{1402}, + 0x54b: []int32{1403}, + 0x54c: []int32{1404}, + 0x54d: []int32{1405}, + 0x54e: []int32{1406}, + 0x54f: []int32{1407}, + 0x550: []int32{1408}, + 0x551: []int32{1409}, + 0x552: []int32{1410}, + 0x553: []int32{1411}, + 0x554: []int32{1412}, + 0x555: []int32{1413}, + 0x556: []int32{1414}, + 0x587: []int32{1381, 1410}, + 0x10a0: []int32{11520}, + 0x10a1: []int32{11521}, + 0x10a2: []int32{11522}, + 0x10a3: []int32{11523}, + 0x10a4: []int32{11524}, + 0x10a5: []int32{11525}, + 0x10a6: []int32{11526}, + 0x10a7: []int32{11527}, + 0x10a8: []int32{11528}, + 0x10a9: []int32{11529}, + 0x10aa: []int32{11530}, + 0x10ab: []int32{11531}, + 0x10ac: []int32{11532}, + 0x10ad: []int32{11533}, + 0x10ae: []int32{11534}, + 0x10af: []int32{11535}, + 0x10b0: []int32{11536}, + 0x10b1: []int32{11537}, + 0x10b2: []int32{11538}, + 0x10b3: []int32{11539}, + 0x10b4: []int32{11540}, + 0x10b5: []int32{11541}, + 0x10b6: []int32{11542}, + 0x10b7: []int32{11543}, + 0x10b8: []int32{11544}, + 0x10b9: []int32{11545}, + 0x10ba: []int32{11546}, + 0x10bb: []int32{11547}, + 0x10bc: []int32{11548}, + 0x10bd: []int32{11549}, + 0x10be: []int32{11550}, + 0x10bf: []int32{11551}, + 0x10c0: []int32{11552}, + 0x10c1: []int32{11553}, + 0x10c2: []int32{11554}, + 0x10c3: []int32{11555}, + 0x10c4: []int32{11556}, + 0x10c5: []int32{11557}, + 0x10c7: []int32{11559}, + 0x10cd: []int32{11565}, + 0x13f8: []int32{5104}, + 0x13f9: []int32{5105}, + 0x13fa: []int32{5106}, + 0x13fb: []int32{5107}, + 0x13fc: []int32{5108}, + 0x13fd: []int32{5109}, + 0x1c80: []int32{1074}, + 0x1c81: []int32{1076}, + 0x1c82: []int32{1086}, + 0x1c83: []int32{1089}, + 0x1c84: []int32{1090}, + 0x1c85: []int32{1090}, + 0x1c86: []int32{1098}, + 0x1c87: []int32{1123}, + 0x1c88: []int32{42571}, + 0x1c90: []int32{4304}, + 0x1c91: []int32{4305}, + 0x1c92: []int32{4306}, + 0x1c93: []int32{4307}, + 0x1c94: []int32{4308}, + 0x1c95: []int32{4309}, + 0x1c96: []int32{4310}, + 0x1c97: []int32{4311}, + 0x1c98: []int32{4312}, + 0x1c99: []int32{4313}, + 0x1c9a: []int32{4314}, + 0x1c9b: []int32{4315}, + 0x1c9c: []int32{4316}, + 0x1c9d: []int32{4317}, + 0x1c9e: []int32{4318}, + 0x1c9f: []int32{4319}, + 0x1ca0: []int32{4320}, + 0x1ca1: []int32{4321}, + 0x1ca2: []int32{4322}, + 0x1ca3: []int32{4323}, + 0x1ca4: []int32{4324}, + 0x1ca5: []int32{4325}, + 0x1ca6: []int32{4326}, + 0x1ca7: []int32{4327}, + 0x1ca8: []int32{4328}, + 0x1ca9: []int32{4329}, + 0x1caa: []int32{4330}, + 0x1cab: []int32{4331}, + 0x1cac: []int32{4332}, + 0x1cad: []int32{4333}, + 0x1cae: []int32{4334}, + 0x1caf: []int32{4335}, + 0x1cb0: []int32{4336}, + 0x1cb1: []int32{4337}, + 0x1cb2: []int32{4338}, + 0x1cb3: []int32{4339}, + 0x1cb4: []int32{4340}, + 0x1cb5: []int32{4341}, + 0x1cb6: []int32{4342}, + 0x1cb7: []int32{4343}, + 0x1cb8: []int32{4344}, + 0x1cb9: []int32{4345}, + 0x1cba: []int32{4346}, + 0x1cbd: []int32{4349}, + 0x1cbe: []int32{4350}, + 0x1cbf: []int32{4351}, + 0x1e00: []int32{7681}, + 0x1e02: []int32{7683}, + 0x1e04: []int32{7685}, + 0x1e06: []int32{7687}, + 0x1e08: []int32{7689}, + 0x1e0a: []int32{7691}, + 0x1e0c: []int32{7693}, + 0x1e0e: []int32{7695}, + 0x1e10: []int32{7697}, + 0x1e12: []int32{7699}, + 0x1e14: []int32{7701}, + 0x1e16: []int32{7703}, + 0x1e18: []int32{7705}, + 0x1e1a: []int32{7707}, + 0x1e1c: []int32{7709}, + 0x1e1e: []int32{7711}, + 0x1e20: []int32{7713}, + 0x1e22: []int32{7715}, + 0x1e24: []int32{7717}, + 0x1e26: []int32{7719}, + 0x1e28: []int32{7721}, + 0x1e2a: []int32{7723}, + 0x1e2c: []int32{7725}, + 0x1e2e: []int32{7727}, + 0x1e30: []int32{7729}, + 0x1e32: []int32{7731}, + 0x1e34: []int32{7733}, + 0x1e36: []int32{7735}, + 0x1e38: []int32{7737}, + 0x1e3a: []int32{7739}, + 0x1e3c: []int32{7741}, + 0x1e3e: []int32{7743}, + 0x1e40: []int32{7745}, + 0x1e42: []int32{7747}, + 0x1e44: []int32{7749}, + 0x1e46: []int32{7751}, + 0x1e48: []int32{7753}, + 0x1e4a: []int32{7755}, + 0x1e4c: []int32{7757}, + 0x1e4e: []int32{7759}, + 0x1e50: []int32{7761}, + 0x1e52: []int32{7763}, + 0x1e54: []int32{7765}, + 0x1e56: []int32{7767}, + 0x1e58: []int32{7769}, + 0x1e5a: []int32{7771}, + 0x1e5c: []int32{7773}, + 0x1e5e: []int32{7775}, + 0x1e60: []int32{7777}, + 0x1e62: []int32{7779}, + 0x1e64: []int32{7781}, + 0x1e66: []int32{7783}, + 0x1e68: []int32{7785}, + 0x1e6a: []int32{7787}, + 0x1e6c: []int32{7789}, + 0x1e6e: []int32{7791}, + 0x1e70: []int32{7793}, + 0x1e72: []int32{7795}, + 0x1e74: []int32{7797}, + 0x1e76: []int32{7799}, + 0x1e78: []int32{7801}, + 0x1e7a: []int32{7803}, + 0x1e7c: []int32{7805}, + 0x1e7e: []int32{7807}, + 0x1e80: []int32{7809}, + 0x1e82: []int32{7811}, + 0x1e84: []int32{7813}, + 0x1e86: []int32{7815}, + 0x1e88: []int32{7817}, + 0x1e8a: []int32{7819}, + 0x1e8c: []int32{7821}, + 0x1e8e: []int32{7823}, + 0x1e90: []int32{7825}, + 0x1e92: []int32{7827}, + 0x1e94: []int32{7829}, + 0x1e96: []int32{104, 817}, + 0x1e97: []int32{116, 776}, + 0x1e98: []int32{119, 778}, + 0x1e99: []int32{121, 778}, + 0x1e9a: []int32{97, 702}, + 0x1e9b: []int32{7777}, + 0x1e9e: []int32{115, 115}, + 0x1ea0: []int32{7841}, + 0x1ea2: []int32{7843}, + 0x1ea4: []int32{7845}, + 0x1ea6: []int32{7847}, + 0x1ea8: []int32{7849}, + 0x1eaa: []int32{7851}, + 0x1eac: []int32{7853}, + 0x1eae: []int32{7855}, + 0x1eb0: []int32{7857}, + 0x1eb2: []int32{7859}, + 0x1eb4: []int32{7861}, + 0x1eb6: []int32{7863}, + 0x1eb8: []int32{7865}, + 0x1eba: []int32{7867}, + 0x1ebc: []int32{7869}, + 0x1ebe: []int32{7871}, + 0x1ec0: []int32{7873}, + 0x1ec2: []int32{7875}, + 0x1ec4: []int32{7877}, + 0x1ec6: []int32{7879}, + 0x1ec8: []int32{7881}, + 0x1eca: []int32{7883}, + 0x1ecc: []int32{7885}, + 0x1ece: []int32{7887}, + 0x1ed0: []int32{7889}, + 0x1ed2: []int32{7891}, + 0x1ed4: []int32{7893}, + 0x1ed6: []int32{7895}, + 0x1ed8: []int32{7897}, + 0x1eda: []int32{7899}, + 0x1edc: []int32{7901}, + 0x1ede: []int32{7903}, + 0x1ee0: []int32{7905}, + 0x1ee2: []int32{7907}, + 0x1ee4: []int32{7909}, + 0x1ee6: []int32{7911}, + 0x1ee8: []int32{7913}, + 0x1eea: []int32{7915}, + 0x1eec: []int32{7917}, + 0x1eee: []int32{7919}, + 0x1ef0: []int32{7921}, + 0x1ef2: []int32{7923}, + 0x1ef4: []int32{7925}, + 0x1ef6: []int32{7927}, + 0x1ef8: []int32{7929}, + 0x1efa: []int32{7931}, + 0x1efc: []int32{7933}, + 0x1efe: []int32{7935}, + 0x1f08: []int32{7936}, + 0x1f09: []int32{7937}, + 0x1f0a: []int32{7938}, + 0x1f0b: []int32{7939}, + 0x1f0c: []int32{7940}, + 0x1f0d: []int32{7941}, + 0x1f0e: []int32{7942}, + 0x1f0f: []int32{7943}, + 0x1f18: []int32{7952}, + 0x1f19: []int32{7953}, + 0x1f1a: []int32{7954}, + 0x1f1b: []int32{7955}, + 0x1f1c: []int32{7956}, + 0x1f1d: []int32{7957}, + 0x1f28: []int32{7968}, + 0x1f29: []int32{7969}, + 0x1f2a: []int32{7970}, + 0x1f2b: []int32{7971}, + 0x1f2c: []int32{7972}, + 0x1f2d: []int32{7973}, + 0x1f2e: []int32{7974}, + 0x1f2f: []int32{7975}, + 0x1f38: []int32{7984}, + 0x1f39: []int32{7985}, + 0x1f3a: []int32{7986}, + 0x1f3b: []int32{7987}, + 0x1f3c: []int32{7988}, + 0x1f3d: []int32{7989}, + 0x1f3e: []int32{7990}, + 0x1f3f: []int32{7991}, + 0x1f48: []int32{8000}, + 0x1f49: []int32{8001}, + 0x1f4a: []int32{8002}, + 0x1f4b: []int32{8003}, + 0x1f4c: []int32{8004}, + 0x1f4d: []int32{8005}, + 0x1f50: []int32{965, 787}, + 0x1f52: []int32{965, 787, 768}, + 0x1f54: []int32{965, 787, 769}, + 0x1f56: []int32{965, 787, 834}, + 0x1f59: []int32{8017}, + 0x1f5b: []int32{8019}, + 0x1f5d: []int32{8021}, + 0x1f5f: []int32{8023}, + 0x1f68: []int32{8032}, + 0x1f69: []int32{8033}, + 0x1f6a: []int32{8034}, + 0x1f6b: []int32{8035}, + 0x1f6c: []int32{8036}, + 0x1f6d: []int32{8037}, + 0x1f6e: []int32{8038}, + 0x1f6f: []int32{8039}, + 0x1f80: []int32{7936, 953}, + 0x1f81: []int32{7937, 953}, + 0x1f82: []int32{7938, 953}, + 0x1f83: []int32{7939, 953}, + 0x1f84: []int32{7940, 953}, + 0x1f85: []int32{7941, 953}, + 0x1f86: []int32{7942, 953}, + 0x1f87: []int32{7943, 953}, + 0x1f88: []int32{7936, 953}, + 0x1f89: []int32{7937, 953}, + 0x1f8a: []int32{7938, 953}, + 0x1f8b: []int32{7939, 953}, + 0x1f8c: []int32{7940, 953}, + 0x1f8d: []int32{7941, 953}, + 0x1f8e: []int32{7942, 953}, + 0x1f8f: []int32{7943, 953}, + 0x1f90: []int32{7968, 953}, + 0x1f91: []int32{7969, 953}, + 0x1f92: []int32{7970, 953}, + 0x1f93: []int32{7971, 953}, + 0x1f94: []int32{7972, 953}, + 0x1f95: []int32{7973, 953}, + 0x1f96: []int32{7974, 953}, + 0x1f97: []int32{7975, 953}, + 0x1f98: []int32{7968, 953}, + 0x1f99: []int32{7969, 953}, + 0x1f9a: []int32{7970, 953}, + 0x1f9b: []int32{7971, 953}, + 0x1f9c: []int32{7972, 953}, + 0x1f9d: []int32{7973, 953}, + 0x1f9e: []int32{7974, 953}, + 0x1f9f: []int32{7975, 953}, + 0x1fa0: []int32{8032, 953}, + 0x1fa1: []int32{8033, 953}, + 0x1fa2: []int32{8034, 953}, + 0x1fa3: []int32{8035, 953}, + 0x1fa4: []int32{8036, 953}, + 0x1fa5: []int32{8037, 953}, + 0x1fa6: []int32{8038, 953}, + 0x1fa7: []int32{8039, 953}, + 0x1fa8: []int32{8032, 953}, + 0x1fa9: []int32{8033, 953}, + 0x1faa: []int32{8034, 953}, + 0x1fab: []int32{8035, 953}, + 0x1fac: []int32{8036, 953}, + 0x1fad: []int32{8037, 953}, + 0x1fae: []int32{8038, 953}, + 0x1faf: []int32{8039, 953}, + 0x1fb2: []int32{8048, 953}, + 0x1fb3: []int32{945, 953}, + 0x1fb4: []int32{940, 953}, + 0x1fb6: []int32{945, 834}, + 0x1fb7: []int32{945, 834, 953}, + 0x1fb8: []int32{8112}, + 0x1fb9: []int32{8113}, + 0x1fba: []int32{8048}, + 0x1fbb: []int32{8049}, + 0x1fbc: []int32{945, 953}, + 0x1fbe: []int32{953}, + 0x1fc2: []int32{8052, 953}, + 0x1fc3: []int32{951, 953}, + 0x1fc4: []int32{942, 953}, + 0x1fc6: []int32{951, 834}, + 0x1fc7: []int32{951, 834, 953}, + 0x1fc8: []int32{8050}, + 0x1fc9: []int32{8051}, + 0x1fca: []int32{8052}, + 0x1fcb: []int32{8053}, + 0x1fcc: []int32{951, 953}, + 0x1fd2: []int32{953, 776, 768}, + 0x1fd3: []int32{953, 776, 769}, + 0x1fd6: []int32{953, 834}, + 0x1fd7: []int32{953, 776, 834}, + 0x1fd8: []int32{8144}, + 0x1fd9: []int32{8145}, + 0x1fda: []int32{8054}, + 0x1fdb: []int32{8055}, + 0x1fe2: []int32{965, 776, 768}, + 0x1fe3: []int32{965, 776, 769}, + 0x1fe4: []int32{961, 787}, + 0x1fe6: []int32{965, 834}, + 0x1fe7: []int32{965, 776, 834}, + 0x1fe8: []int32{8160}, + 0x1fe9: []int32{8161}, + 0x1fea: []int32{8058}, + 0x1feb: []int32{8059}, + 0x1fec: []int32{8165}, + 0x1ff2: []int32{8060, 953}, + 0x1ff3: []int32{969, 953}, + 0x1ff4: []int32{974, 953}, + 0x1ff6: []int32{969, 834}, + 0x1ff7: []int32{969, 834, 953}, + 0x1ff8: []int32{8056}, + 0x1ff9: []int32{8057}, + 0x1ffa: []int32{8060}, + 0x1ffb: []int32{8061}, + 0x1ffc: []int32{969, 953}, + 0x2126: []int32{969}, + 0x212a: []int32{107}, + 0x212b: []int32{229}, + 0x2132: []int32{8526}, + 0x2160: []int32{8560}, + 0x2161: []int32{8561}, + 0x2162: []int32{8562}, + 0x2163: []int32{8563}, + 0x2164: []int32{8564}, + 0x2165: []int32{8565}, + 0x2166: []int32{8566}, + 0x2167: []int32{8567}, + 0x2168: []int32{8568}, + 0x2169: []int32{8569}, + 0x216a: []int32{8570}, + 0x216b: []int32{8571}, + 0x216c: []int32{8572}, + 0x216d: []int32{8573}, + 0x216e: []int32{8574}, + 0x216f: []int32{8575}, + 0x2183: []int32{8580}, + 0x24b6: []int32{9424}, + 0x24b7: []int32{9425}, + 0x24b8: []int32{9426}, + 0x24b9: []int32{9427}, + 0x24ba: []int32{9428}, + 0x24bb: []int32{9429}, + 0x24bc: []int32{9430}, + 0x24bd: []int32{9431}, + 0x24be: []int32{9432}, + 0x24bf: []int32{9433}, + 0x24c0: []int32{9434}, + 0x24c1: []int32{9435}, + 0x24c2: []int32{9436}, + 0x24c3: []int32{9437}, + 0x24c4: []int32{9438}, + 0x24c5: []int32{9439}, + 0x24c6: []int32{9440}, + 0x24c7: []int32{9441}, + 0x24c8: []int32{9442}, + 0x24c9: []int32{9443}, + 0x24ca: []int32{9444}, + 0x24cb: []int32{9445}, + 0x24cc: []int32{9446}, + 0x24cd: []int32{9447}, + 0x24ce: []int32{9448}, + 0x24cf: []int32{9449}, + 0x2c00: []int32{11312}, + 0x2c01: []int32{11313}, + 0x2c02: []int32{11314}, + 0x2c03: []int32{11315}, + 0x2c04: []int32{11316}, + 0x2c05: []int32{11317}, + 0x2c06: []int32{11318}, + 0x2c07: []int32{11319}, + 0x2c08: []int32{11320}, + 0x2c09: []int32{11321}, + 0x2c0a: []int32{11322}, + 0x2c0b: []int32{11323}, + 0x2c0c: []int32{11324}, + 0x2c0d: []int32{11325}, + 0x2c0e: []int32{11326}, + 0x2c0f: []int32{11327}, + 0x2c10: []int32{11328}, + 0x2c11: []int32{11329}, + 0x2c12: []int32{11330}, + 0x2c13: []int32{11331}, + 0x2c14: []int32{11332}, + 0x2c15: []int32{11333}, + 0x2c16: []int32{11334}, + 0x2c17: []int32{11335}, + 0x2c18: []int32{11336}, + 0x2c19: []int32{11337}, + 0x2c1a: []int32{11338}, + 0x2c1b: []int32{11339}, + 0x2c1c: []int32{11340}, + 0x2c1d: []int32{11341}, + 0x2c1e: []int32{11342}, + 0x2c1f: []int32{11343}, + 0x2c20: []int32{11344}, + 0x2c21: []int32{11345}, + 0x2c22: []int32{11346}, + 0x2c23: []int32{11347}, + 0x2c24: []int32{11348}, + 0x2c25: []int32{11349}, + 0x2c26: []int32{11350}, + 0x2c27: []int32{11351}, + 0x2c28: []int32{11352}, + 0x2c29: []int32{11353}, + 0x2c2a: []int32{11354}, + 0x2c2b: []int32{11355}, + 0x2c2c: []int32{11356}, + 0x2c2d: []int32{11357}, + 0x2c2e: []int32{11358}, + 0x2c60: []int32{11361}, + 0x2c62: []int32{619}, + 0x2c63: []int32{7549}, + 0x2c64: []int32{637}, + 0x2c67: []int32{11368}, + 0x2c69: []int32{11370}, + 0x2c6b: []int32{11372}, + 0x2c6d: []int32{593}, + 0x2c6e: []int32{625}, + 0x2c6f: []int32{592}, + 0x2c70: []int32{594}, + 0x2c72: []int32{11379}, + 0x2c75: []int32{11382}, + 0x2c7e: []int32{575}, + 0x2c7f: []int32{576}, + 0x2c80: []int32{11393}, + 0x2c82: []int32{11395}, + 0x2c84: []int32{11397}, + 0x2c86: []int32{11399}, + 0x2c88: []int32{11401}, + 0x2c8a: []int32{11403}, + 0x2c8c: []int32{11405}, + 0x2c8e: []int32{11407}, + 0x2c90: []int32{11409}, + 0x2c92: []int32{11411}, + 0x2c94: []int32{11413}, + 0x2c96: []int32{11415}, + 0x2c98: []int32{11417}, + 0x2c9a: []int32{11419}, + 0x2c9c: []int32{11421}, + 0x2c9e: []int32{11423}, + 0x2ca0: []int32{11425}, + 0x2ca2: []int32{11427}, + 0x2ca4: []int32{11429}, + 0x2ca6: []int32{11431}, + 0x2ca8: []int32{11433}, + 0x2caa: []int32{11435}, + 0x2cac: []int32{11437}, + 0x2cae: []int32{11439}, + 0x2cb0: []int32{11441}, + 0x2cb2: []int32{11443}, + 0x2cb4: []int32{11445}, + 0x2cb6: []int32{11447}, + 0x2cb8: []int32{11449}, + 0x2cba: []int32{11451}, + 0x2cbc: []int32{11453}, + 0x2cbe: []int32{11455}, + 0x2cc0: []int32{11457}, + 0x2cc2: []int32{11459}, + 0x2cc4: []int32{11461}, + 0x2cc6: []int32{11463}, + 0x2cc8: []int32{11465}, + 0x2cca: []int32{11467}, + 0x2ccc: []int32{11469}, + 0x2cce: []int32{11471}, + 0x2cd0: []int32{11473}, + 0x2cd2: []int32{11475}, + 0x2cd4: []int32{11477}, + 0x2cd6: []int32{11479}, + 0x2cd8: []int32{11481}, + 0x2cda: []int32{11483}, + 0x2cdc: []int32{11485}, + 0x2cde: []int32{11487}, + 0x2ce0: []int32{11489}, + 0x2ce2: []int32{11491}, + 0x2ceb: []int32{11500}, + 0x2ced: []int32{11502}, + 0x2cf2: []int32{11507}, + 0xa640: []int32{42561}, + 0xa642: []int32{42563}, + 0xa644: []int32{42565}, + 0xa646: []int32{42567}, + 0xa648: []int32{42569}, + 0xa64a: []int32{42571}, + 0xa64c: []int32{42573}, + 0xa64e: []int32{42575}, + 0xa650: []int32{42577}, + 0xa652: []int32{42579}, + 0xa654: []int32{42581}, + 0xa656: []int32{42583}, + 0xa658: []int32{42585}, + 0xa65a: []int32{42587}, + 0xa65c: []int32{42589}, + 0xa65e: []int32{42591}, + 0xa660: []int32{42593}, + 0xa662: []int32{42595}, + 0xa664: []int32{42597}, + 0xa666: []int32{42599}, + 0xa668: []int32{42601}, + 0xa66a: []int32{42603}, + 0xa66c: []int32{42605}, + 0xa680: []int32{42625}, + 0xa682: []int32{42627}, + 0xa684: []int32{42629}, + 0xa686: []int32{42631}, + 0xa688: []int32{42633}, + 0xa68a: []int32{42635}, + 0xa68c: []int32{42637}, + 0xa68e: []int32{42639}, + 0xa690: []int32{42641}, + 0xa692: []int32{42643}, + 0xa694: []int32{42645}, + 0xa696: []int32{42647}, + 0xa698: []int32{42649}, + 0xa69a: []int32{42651}, + 0xa722: []int32{42787}, + 0xa724: []int32{42789}, + 0xa726: []int32{42791}, + 0xa728: []int32{42793}, + 0xa72a: []int32{42795}, + 0xa72c: []int32{42797}, + 0xa72e: []int32{42799}, + 0xa732: []int32{42803}, + 0xa734: []int32{42805}, + 0xa736: []int32{42807}, + 0xa738: []int32{42809}, + 0xa73a: []int32{42811}, + 0xa73c: []int32{42813}, + 0xa73e: []int32{42815}, + 0xa740: []int32{42817}, + 0xa742: []int32{42819}, + 0xa744: []int32{42821}, + 0xa746: []int32{42823}, + 0xa748: []int32{42825}, + 0xa74a: []int32{42827}, + 0xa74c: []int32{42829}, + 0xa74e: []int32{42831}, + 0xa750: []int32{42833}, + 0xa752: []int32{42835}, + 0xa754: []int32{42837}, + 0xa756: []int32{42839}, + 0xa758: []int32{42841}, + 0xa75a: []int32{42843}, + 0xa75c: []int32{42845}, + 0xa75e: []int32{42847}, + 0xa760: []int32{42849}, + 0xa762: []int32{42851}, + 0xa764: []int32{42853}, + 0xa766: []int32{42855}, + 0xa768: []int32{42857}, + 0xa76a: []int32{42859}, + 0xa76c: []int32{42861}, + 0xa76e: []int32{42863}, + 0xa779: []int32{42874}, + 0xa77b: []int32{42876}, + 0xa77d: []int32{7545}, + 0xa77e: []int32{42879}, + 0xa780: []int32{42881}, + 0xa782: []int32{42883}, + 0xa784: []int32{42885}, + 0xa786: []int32{42887}, + 0xa78b: []int32{42892}, + 0xa78d: []int32{613}, + 0xa790: []int32{42897}, + 0xa792: []int32{42899}, + 0xa796: []int32{42903}, + 0xa798: []int32{42905}, + 0xa79a: []int32{42907}, + 0xa79c: []int32{42909}, + 0xa79e: []int32{42911}, + 0xa7a0: []int32{42913}, + 0xa7a2: []int32{42915}, + 0xa7a4: []int32{42917}, + 0xa7a6: []int32{42919}, + 0xa7a8: []int32{42921}, + 0xa7aa: []int32{614}, + 0xa7ab: []int32{604}, + 0xa7ac: []int32{609}, + 0xa7ad: []int32{620}, + 0xa7ae: []int32{618}, + 0xa7b0: []int32{670}, + 0xa7b1: []int32{647}, + 0xa7b2: []int32{669}, + 0xa7b3: []int32{43859}, + 0xa7b4: []int32{42933}, + 0xa7b6: []int32{42935}, + 0xa7b8: []int32{42937}, + 0xa7ba: []int32{42939}, + 0xa7bc: []int32{42941}, + 0xa7be: []int32{42943}, + 0xa7c2: []int32{42947}, + 0xa7c4: []int32{42900}, + 0xa7c5: []int32{642}, + 0xa7c6: []int32{7566}, + 0xab70: []int32{5024}, + 0xab71: []int32{5025}, + 0xab72: []int32{5026}, + 0xab73: []int32{5027}, + 0xab74: []int32{5028}, + 0xab75: []int32{5029}, + 0xab76: []int32{5030}, + 0xab77: []int32{5031}, + 0xab78: []int32{5032}, + 0xab79: []int32{5033}, + 0xab7a: []int32{5034}, + 0xab7b: []int32{5035}, + 0xab7c: []int32{5036}, + 0xab7d: []int32{5037}, + 0xab7e: []int32{5038}, + 0xab7f: []int32{5039}, + 0xab80: []int32{5040}, + 0xab81: []int32{5041}, + 0xab82: []int32{5042}, + 0xab83: []int32{5043}, + 0xab84: []int32{5044}, + 0xab85: []int32{5045}, + 0xab86: []int32{5046}, + 0xab87: []int32{5047}, + 0xab88: []int32{5048}, + 0xab89: []int32{5049}, + 0xab8a: []int32{5050}, + 0xab8b: []int32{5051}, + 0xab8c: []int32{5052}, + 0xab8d: []int32{5053}, + 0xab8e: []int32{5054}, + 0xab8f: []int32{5055}, + 0xab90: []int32{5056}, + 0xab91: []int32{5057}, + 0xab92: []int32{5058}, + 0xab93: []int32{5059}, + 0xab94: []int32{5060}, + 0xab95: []int32{5061}, + 0xab96: []int32{5062}, + 0xab97: []int32{5063}, + 0xab98: []int32{5064}, + 0xab99: []int32{5065}, + 0xab9a: []int32{5066}, + 0xab9b: []int32{5067}, + 0xab9c: []int32{5068}, + 0xab9d: []int32{5069}, + 0xab9e: []int32{5070}, + 0xab9f: []int32{5071}, + 0xaba0: []int32{5072}, + 0xaba1: []int32{5073}, + 0xaba2: []int32{5074}, + 0xaba3: []int32{5075}, + 0xaba4: []int32{5076}, + 0xaba5: []int32{5077}, + 0xaba6: []int32{5078}, + 0xaba7: []int32{5079}, + 0xaba8: []int32{5080}, + 0xaba9: []int32{5081}, + 0xabaa: []int32{5082}, + 0xabab: []int32{5083}, + 0xabac: []int32{5084}, + 0xabad: []int32{5085}, + 0xabae: []int32{5086}, + 0xabaf: []int32{5087}, + 0xabb0: []int32{5088}, + 0xabb1: []int32{5089}, + 0xabb2: []int32{5090}, + 0xabb3: []int32{5091}, + 0xabb4: []int32{5092}, + 0xabb5: []int32{5093}, + 0xabb6: []int32{5094}, + 0xabb7: []int32{5095}, + 0xabb8: []int32{5096}, + 0xabb9: []int32{5097}, + 0xabba: []int32{5098}, + 0xabbb: []int32{5099}, + 0xabbc: []int32{5100}, + 0xabbd: []int32{5101}, + 0xabbe: []int32{5102}, + 0xabbf: []int32{5103}, + 0xfb00: []int32{102, 102}, + 0xfb01: []int32{102, 105}, + 0xfb02: []int32{102, 108}, + 0xfb03: []int32{102, 102, 105}, + 0xfb04: []int32{102, 102, 108}, + 0xfb05: []int32{115, 116}, + 0xfb06: []int32{115, 116}, + 0xfb13: []int32{1396, 1398}, + 0xfb14: []int32{1396, 1381}, + 0xfb15: []int32{1396, 1387}, + 0xfb16: []int32{1406, 1398}, + 0xfb17: []int32{1396, 1389}, + 0xff21: []int32{65345}, + 0xff22: []int32{65346}, + 0xff23: []int32{65347}, + 0xff24: []int32{65348}, + 0xff25: []int32{65349}, + 0xff26: []int32{65350}, + 0xff27: []int32{65351}, + 0xff28: []int32{65352}, + 0xff29: []int32{65353}, + 0xff2a: []int32{65354}, + 0xff2b: []int32{65355}, + 0xff2c: []int32{65356}, + 0xff2d: []int32{65357}, + 0xff2e: []int32{65358}, + 0xff2f: []int32{65359}, + 0xff30: []int32{65360}, + 0xff31: []int32{65361}, + 0xff32: []int32{65362}, + 0xff33: []int32{65363}, + 0xff34: []int32{65364}, + 0xff35: []int32{65365}, + 0xff36: []int32{65366}, + 0xff37: []int32{65367}, + 0xff38: []int32{65368}, + 0xff39: []int32{65369}, + 0xff3a: []int32{65370}, + 0x10400: []int32{66600}, + 0x10401: []int32{66601}, + 0x10402: []int32{66602}, + 0x10403: []int32{66603}, + 0x10404: []int32{66604}, + 0x10405: []int32{66605}, + 0x10406: []int32{66606}, + 0x10407: []int32{66607}, + 0x10408: []int32{66608}, + 0x10409: []int32{66609}, + 0x1040a: []int32{66610}, + 0x1040b: []int32{66611}, + 0x1040c: []int32{66612}, + 0x1040d: []int32{66613}, + 0x1040e: []int32{66614}, + 0x1040f: []int32{66615}, + 0x10410: []int32{66616}, + 0x10411: []int32{66617}, + 0x10412: []int32{66618}, + 0x10413: []int32{66619}, + 0x10414: []int32{66620}, + 0x10415: []int32{66621}, + 0x10416: []int32{66622}, + 0x10417: []int32{66623}, + 0x10418: []int32{66624}, + 0x10419: []int32{66625}, + 0x1041a: []int32{66626}, + 0x1041b: []int32{66627}, + 0x1041c: []int32{66628}, + 0x1041d: []int32{66629}, + 0x1041e: []int32{66630}, + 0x1041f: []int32{66631}, + 0x10420: []int32{66632}, + 0x10421: []int32{66633}, + 0x10422: []int32{66634}, + 0x10423: []int32{66635}, + 0x10424: []int32{66636}, + 0x10425: []int32{66637}, + 0x10426: []int32{66638}, + 0x10427: []int32{66639}, + 0x104b0: []int32{66776}, + 0x104b1: []int32{66777}, + 0x104b2: []int32{66778}, + 0x104b3: []int32{66779}, + 0x104b4: []int32{66780}, + 0x104b5: []int32{66781}, + 0x104b6: []int32{66782}, + 0x104b7: []int32{66783}, + 0x104b8: []int32{66784}, + 0x104b9: []int32{66785}, + 0x104ba: []int32{66786}, + 0x104bb: []int32{66787}, + 0x104bc: []int32{66788}, + 0x104bd: []int32{66789}, + 0x104be: []int32{66790}, + 0x104bf: []int32{66791}, + 0x104c0: []int32{66792}, + 0x104c1: []int32{66793}, + 0x104c2: []int32{66794}, + 0x104c3: []int32{66795}, + 0x104c4: []int32{66796}, + 0x104c5: []int32{66797}, + 0x104c6: []int32{66798}, + 0x104c7: []int32{66799}, + 0x104c8: []int32{66800}, + 0x104c9: []int32{66801}, + 0x104ca: []int32{66802}, + 0x104cb: []int32{66803}, + 0x104cc: []int32{66804}, + 0x104cd: []int32{66805}, + 0x104ce: []int32{66806}, + 0x104cf: []int32{66807}, + 0x104d0: []int32{66808}, + 0x104d1: []int32{66809}, + 0x104d2: []int32{66810}, + 0x104d3: []int32{66811}, + 0x10c80: []int32{68800}, + 0x10c81: []int32{68801}, + 0x10c82: []int32{68802}, + 0x10c83: []int32{68803}, + 0x10c84: []int32{68804}, + 0x10c85: []int32{68805}, + 0x10c86: []int32{68806}, + 0x10c87: []int32{68807}, + 0x10c88: []int32{68808}, + 0x10c89: []int32{68809}, + 0x10c8a: []int32{68810}, + 0x10c8b: []int32{68811}, + 0x10c8c: []int32{68812}, + 0x10c8d: []int32{68813}, + 0x10c8e: []int32{68814}, + 0x10c8f: []int32{68815}, + 0x10c90: []int32{68816}, + 0x10c91: []int32{68817}, + 0x10c92: []int32{68818}, + 0x10c93: []int32{68819}, + 0x10c94: []int32{68820}, + 0x10c95: []int32{68821}, + 0x10c96: []int32{68822}, + 0x10c97: []int32{68823}, + 0x10c98: []int32{68824}, + 0x10c99: []int32{68825}, + 0x10c9a: []int32{68826}, + 0x10c9b: []int32{68827}, + 0x10c9c: []int32{68828}, + 0x10c9d: []int32{68829}, + 0x10c9e: []int32{68830}, + 0x10c9f: []int32{68831}, + 0x10ca0: []int32{68832}, + 0x10ca1: []int32{68833}, + 0x10ca2: []int32{68834}, + 0x10ca3: []int32{68835}, + 0x10ca4: []int32{68836}, + 0x10ca5: []int32{68837}, + 0x10ca6: []int32{68838}, + 0x10ca7: []int32{68839}, + 0x10ca8: []int32{68840}, + 0x10ca9: []int32{68841}, + 0x10caa: []int32{68842}, + 0x10cab: []int32{68843}, + 0x10cac: []int32{68844}, + 0x10cad: []int32{68845}, + 0x10cae: []int32{68846}, + 0x10caf: []int32{68847}, + 0x10cb0: []int32{68848}, + 0x10cb1: []int32{68849}, + 0x10cb2: []int32{68850}, + 0x118a0: []int32{71872}, + 0x118a1: []int32{71873}, + 0x118a2: []int32{71874}, + 0x118a3: []int32{71875}, + 0x118a4: []int32{71876}, + 0x118a5: []int32{71877}, + 0x118a6: []int32{71878}, + 0x118a7: []int32{71879}, + 0x118a8: []int32{71880}, + 0x118a9: []int32{71881}, + 0x118aa: []int32{71882}, + 0x118ab: []int32{71883}, + 0x118ac: []int32{71884}, + 0x118ad: []int32{71885}, + 0x118ae: []int32{71886}, + 0x118af: []int32{71887}, + 0x118b0: []int32{71888}, + 0x118b1: []int32{71889}, + 0x118b2: []int32{71890}, + 0x118b3: []int32{71891}, + 0x118b4: []int32{71892}, + 0x118b5: []int32{71893}, + 0x118b6: []int32{71894}, + 0x118b7: []int32{71895}, + 0x118b8: []int32{71896}, + 0x118b9: []int32{71897}, + 0x118ba: []int32{71898}, + 0x118bb: []int32{71899}, + 0x118bc: []int32{71900}, + 0x118bd: []int32{71901}, + 0x118be: []int32{71902}, + 0x118bf: []int32{71903}, + 0x16e40: []int32{93792}, + 0x16e41: []int32{93793}, + 0x16e42: []int32{93794}, + 0x16e43: []int32{93795}, + 0x16e44: []int32{93796}, + 0x16e45: []int32{93797}, + 0x16e46: []int32{93798}, + 0x16e47: []int32{93799}, + 0x16e48: []int32{93800}, + 0x16e49: []int32{93801}, + 0x16e4a: []int32{93802}, + 0x16e4b: []int32{93803}, + 0x16e4c: []int32{93804}, + 0x16e4d: []int32{93805}, + 0x16e4e: []int32{93806}, + 0x16e4f: []int32{93807}, + 0x16e50: []int32{93808}, + 0x16e51: []int32{93809}, + 0x16e52: []int32{93810}, + 0x16e53: []int32{93811}, + 0x16e54: []int32{93812}, + 0x16e55: []int32{93813}, + 0x16e56: []int32{93814}, + 0x16e57: []int32{93815}, + 0x16e58: []int32{93816}, + 0x16e59: []int32{93817}, + 0x16e5a: []int32{93818}, + 0x16e5b: []int32{93819}, + 0x16e5c: []int32{93820}, + 0x16e5d: []int32{93821}, + 0x16e5e: []int32{93822}, + 0x16e5f: []int32{93823}, + 0x1e900: []int32{125218}, + 0x1e901: []int32{125219}, + 0x1e902: []int32{125220}, + 0x1e903: []int32{125221}, + 0x1e904: []int32{125222}, + 0x1e905: []int32{125223}, + 0x1e906: []int32{125224}, + 0x1e907: []int32{125225}, + 0x1e908: []int32{125226}, + 0x1e909: []int32{125227}, + 0x1e90a: []int32{125228}, + 0x1e90b: []int32{125229}, + 0x1e90c: []int32{125230}, + 0x1e90d: []int32{125231}, + 0x1e90e: []int32{125232}, + 0x1e90f: []int32{125233}, + 0x1e910: []int32{125234}, + 0x1e911: []int32{125235}, + 0x1e912: []int32{125236}, + 0x1e913: []int32{125237}, + 0x1e914: []int32{125238}, + 0x1e915: []int32{125239}, + 0x1e916: []int32{125240}, + 0x1e917: []int32{125241}, + 0x1e918: []int32{125242}, + 0x1e919: []int32{125243}, + 0x1e91a: []int32{125244}, + 0x1e91b: []int32{125245}, + 0x1e91c: []int32{125246}, + 0x1e91d: []int32{125247}, + 0x1e91e: []int32{125248}, + 0x1e91f: []int32{125249}, + 0x1e920: []int32{125250}, + 0x1e921: []int32{125251}, +} diff --git a/vendor/github.com/yuin/goldmark/util/util.go b/vendor/github.com/yuin/goldmark/util/util.go new file mode 100644 index 00000000..a817ec63 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/util/util.go @@ -0,0 +1,980 @@ +// Package util provides utility functions for the goldmark. +package util + +import ( + "bytes" + "io" + "net/url" + "regexp" + "sort" + "strconv" + "unicode" + "unicode/utf8" +) + +// A CopyOnWriteBuffer is a byte buffer that copies buffer when +// it need to be changed. +type CopyOnWriteBuffer struct { + buffer []byte + copied bool +} + +// NewCopyOnWriteBuffer returns a new CopyOnWriteBuffer. +func NewCopyOnWriteBuffer(buffer []byte) CopyOnWriteBuffer { + return CopyOnWriteBuffer{ + buffer: buffer, + copied: false, + } +} + +// Write writes given bytes to the buffer. +// Write allocate new buffer and clears it at the first time. +func (b *CopyOnWriteBuffer) Write(value []byte) { + if !b.copied { + b.buffer = make([]byte, 0, len(b.buffer)+20) + b.copied = true + } + b.buffer = append(b.buffer, value...) +} + +// WriteString writes given string to the buffer. +// WriteString allocate new buffer and clears it at the first time. +func (b *CopyOnWriteBuffer) WriteString(value string) { + b.Write(StringToReadOnlyBytes(value)) +} + +// Append appends given bytes to the buffer. +// Append copy buffer at the first time. +func (b *CopyOnWriteBuffer) Append(value []byte) { + if !b.copied { + tmp := make([]byte, len(b.buffer), len(b.buffer)+20) + copy(tmp, b.buffer) + b.buffer = tmp + b.copied = true + } + b.buffer = append(b.buffer, value...) +} + +// AppendString appends given string to the buffer. +// AppendString copy buffer at the first time. +func (b *CopyOnWriteBuffer) AppendString(value string) { + b.Append(StringToReadOnlyBytes(value)) +} + +// WriteByte writes the given byte to the buffer. +// WriteByte allocate new buffer and clears it at the first time. +func (b *CopyOnWriteBuffer) WriteByte(c byte) { + if !b.copied { + b.buffer = make([]byte, 0, len(b.buffer)+20) + b.copied = true + } + b.buffer = append(b.buffer, c) +} + +// AppendByte appends given bytes to the buffer. +// AppendByte copy buffer at the first time. +func (b *CopyOnWriteBuffer) AppendByte(c byte) { + if !b.copied { + tmp := make([]byte, len(b.buffer), len(b.buffer)+20) + copy(tmp, b.buffer) + b.buffer = tmp + b.copied = true + } + b.buffer = append(b.buffer, c) +} + +// Bytes returns bytes of this buffer. +func (b *CopyOnWriteBuffer) Bytes() []byte { + return b.buffer +} + +// IsCopied returns true if buffer has been copied, otherwise false. +func (b *CopyOnWriteBuffer) IsCopied() bool { + return b.copied +} + +// IsEscapedPunctuation returns true if character at a given index i +// is an escaped punctuation, otherwise false. +func IsEscapedPunctuation(source []byte, i int) bool { + return source[i] == '\\' && i < len(source)-1 && IsPunct(source[i+1]) +} + +// ReadWhile read the given source while pred is true. +func ReadWhile(source []byte, index [2]int, pred func(byte) bool) (int, bool) { + j := index[0] + ok := false + for ; j < index[1]; j++ { + c1 := source[j] + if pred(c1) { + ok = true + continue + } + break + } + return j, ok +} + +// IsBlank returns true if the given string is all space characters. +func IsBlank(bs []byte) bool { + for _, b := range bs { + if !IsSpace(b) { + return false + } + } + return true +} + +// VisualizeSpaces visualize invisible space characters. +func VisualizeSpaces(bs []byte) []byte { + bs = bytes.Replace(bs, []byte(" "), []byte("[SPACE]"), -1) + bs = bytes.Replace(bs, []byte("\t"), []byte("[TAB]"), -1) + bs = bytes.Replace(bs, []byte("\n"), []byte("[NEWLINE]\n"), -1) + bs = bytes.Replace(bs, []byte("\r"), []byte("[CR]"), -1) + bs = bytes.Replace(bs, []byte("\v"), []byte("[VTAB]"), -1) + bs = bytes.Replace(bs, []byte("\x00"), []byte("[NUL]"), -1) + bs = bytes.Replace(bs, []byte("\ufffd"), []byte("[U+FFFD]"), -1) + return bs +} + +// TabWidth calculates actual width of a tab at the given position. +func TabWidth(currentPos int) int { + return 4 - currentPos%4 +} + +// IndentPosition searches an indent position with the given width for the given line. +// If the line contains tab characters, paddings may be not zero. +// currentPos==0 and width==2: +// +// position: 0 1 +// [TAB]aaaa +// width: 1234 5678 +// +// width=2 is in the tab character. In this case, IndentPosition returns +// (pos=1, padding=2) +func IndentPosition(bs []byte, currentPos, width int) (pos, padding int) { + return IndentPositionPadding(bs, currentPos, 0, width) +} + +// IndentPositionPadding searches an indent position with the given width for the given line. +// This function is mostly same as IndentPosition except this function +// takes account into additional paddings. +func IndentPositionPadding(bs []byte, currentPos, paddingv, width int) (pos, padding int) { + if width == 0 { + return 0, paddingv + } + w := 0 + i := 0 + l := len(bs) + for ; i < l; i++ { + if bs[i] == '\t' && w < width { + w += TabWidth(currentPos + w) + } else if bs[i] == ' ' && w < width { + w++ + } else { + break + } + } + if w >= width { + return i - paddingv, w - width + } + return -1, -1 +} + +// DedentPosition dedents lines by the given width. +// +// Deprecated: This function has bugs. Use util.IndentPositionPadding and util.FirstNonSpacePosition. +func DedentPosition(bs []byte, currentPos, width int) (pos, padding int) { + if width == 0 { + return 0, 0 + } + w := 0 + l := len(bs) + i := 0 + for ; i < l; i++ { + if bs[i] == '\t' { + w += TabWidth(currentPos + w) + } else if bs[i] == ' ' { + w++ + } else { + break + } + } + if w >= width { + return i, w - width + } + return i, 0 +} + +// DedentPositionPadding dedents lines by the given width. +// This function is mostly same as DedentPosition except this function +// takes account into additional paddings. +// +// Deprecated: This function has bugs. Use util.IndentPositionPadding and util.FirstNonSpacePosition. +func DedentPositionPadding(bs []byte, currentPos, paddingv, width int) (pos, padding int) { + if width == 0 { + return 0, paddingv + } + + w := 0 + i := 0 + l := len(bs) + for ; i < l; i++ { + if bs[i] == '\t' { + w += TabWidth(currentPos + w) + } else if bs[i] == ' ' { + w++ + } else { + break + } + } + if w >= width { + return i - paddingv, w - width + } + return i - paddingv, 0 +} + +// IndentWidth calculate an indent width for the given line. +func IndentWidth(bs []byte, currentPos int) (width, pos int) { + l := len(bs) + for i := 0; i < l; i++ { + b := bs[i] + if b == ' ' { + width++ + pos++ + } else if b == '\t' { + width += TabWidth(currentPos + width) + pos++ + } else { + break + } + } + return +} + +// FirstNonSpacePosition returns a position line that is a first nonspace +// character. +func FirstNonSpacePosition(bs []byte) int { + i := 0 + for ; i < len(bs); i++ { + c := bs[i] + if c == ' ' || c == '\t' { + continue + } + if c == '\n' { + return -1 + } + return i + } + return -1 +} + +// FindClosure returns a position that closes the given opener. +// If codeSpan is set true, it ignores characters in code spans. +// If allowNesting is set true, closures correspond to nested opener will be +// ignored. +// +// Deprecated: This function can not handle newlines. Many elements +// can be existed over multiple lines(e.g. link labels). +// Use text.Reader.FindClosure. +func FindClosure(bs []byte, opener, closure byte, codeSpan, allowNesting bool) int { + i := 0 + opened := 1 + codeSpanOpener := 0 + for i < len(bs) { + c := bs[i] + if codeSpan && codeSpanOpener != 0 && c == '`' { + codeSpanCloser := 0 + for ; i < len(bs); i++ { + if bs[i] == '`' { + codeSpanCloser++ + } else { + i-- + break + } + } + if codeSpanCloser == codeSpanOpener { + codeSpanOpener = 0 + } + } else if codeSpanOpener == 0 && c == '\\' && i < len(bs)-1 && IsPunct(bs[i+1]) { + i += 2 + continue + } else if codeSpan && codeSpanOpener == 0 && c == '`' { + for ; i < len(bs); i++ { + if bs[i] == '`' { + codeSpanOpener++ + } else { + i-- + break + } + } + } else if (codeSpan && codeSpanOpener == 0) || !codeSpan { + if c == closure { + opened-- + if opened == 0 { + return i + } + } else if c == opener { + if !allowNesting { + return -1 + } + opened++ + } + } + i++ + } + return -1 +} + +// TrimLeft trims characters in the given s from head of the source. +// bytes.TrimLeft offers same functionalities, but bytes.TrimLeft +// allocates new buffer for the result. +func TrimLeft(source, b []byte) []byte { + i := 0 + for ; i < len(source); i++ { + c := source[i] + found := false + for j := 0; j < len(b); j++ { + if c == b[j] { + found = true + break + } + } + if !found { + break + } + } + return source[i:] +} + +// TrimRight trims characters in the given s from tail of the source. +func TrimRight(source, b []byte) []byte { + i := len(source) - 1 + for ; i >= 0; i-- { + c := source[i] + found := false + for j := 0; j < len(b); j++ { + if c == b[j] { + found = true + break + } + } + if !found { + break + } + } + return source[:i+1] +} + +// TrimLeftLength returns a length of leading specified characters. +func TrimLeftLength(source, s []byte) int { + return len(source) - len(TrimLeft(source, s)) +} + +// TrimRightLength returns a length of trailing specified characters. +func TrimRightLength(source, s []byte) int { + return len(source) - len(TrimRight(source, s)) +} + +// TrimLeftSpaceLength returns a length of leading space characters. +func TrimLeftSpaceLength(source []byte) int { + i := 0 + for ; i < len(source); i++ { + if !IsSpace(source[i]) { + break + } + } + return i +} + +// TrimRightSpaceLength returns a length of trailing space characters. +func TrimRightSpaceLength(source []byte) int { + l := len(source) + i := l - 1 + for ; i >= 0; i-- { + if !IsSpace(source[i]) { + break + } + } + if i < 0 { + return l + } + return l - 1 - i +} + +// TrimLeftSpace returns a subslice of the given string by slicing off all leading +// space characters. +func TrimLeftSpace(source []byte) []byte { + return TrimLeft(source, spaces) +} + +// TrimRightSpace returns a subslice of the given string by slicing off all trailing +// space characters. +func TrimRightSpace(source []byte) []byte { + return TrimRight(source, spaces) +} + +// DoFullUnicodeCaseFolding performs full unicode case folding to given bytes. +func DoFullUnicodeCaseFolding(v []byte) []byte { + var rbuf []byte + cob := NewCopyOnWriteBuffer(v) + n := 0 + for i := 0; i < len(v); i++ { + c := v[i] + if c < 0xb5 { + if c >= 0x41 && c <= 0x5a { + // A-Z to a-z + cob.Write(v[n:i]) + cob.WriteByte(c + 32) + n = i + 1 + } + continue + } + + if !utf8.RuneStart(c) { + continue + } + r, length := utf8.DecodeRune(v[i:]) + if r == utf8.RuneError { + continue + } + folded, ok := unicodeCaseFoldings[r] + if !ok { + continue + } + + cob.Write(v[n:i]) + if rbuf == nil { + rbuf = make([]byte, 4) + } + for _, f := range folded { + l := utf8.EncodeRune(rbuf, f) + cob.Write(rbuf[:l]) + } + i += length - 1 + n = i + 1 + } + if cob.IsCopied() { + cob.Write(v[n:]) + } + return cob.Bytes() +} + +// ReplaceSpaces replaces sequence of spaces with the given repl. +func ReplaceSpaces(source []byte, repl byte) []byte { + var ret []byte + start := -1 + for i, c := range source { + iss := IsSpace(c) + if start < 0 && iss { + start = i + continue + } else if start >= 0 && iss { + continue + } else if start >= 0 { + if ret == nil { + ret = make([]byte, 0, len(source)) + ret = append(ret, source[:start]...) + } + ret = append(ret, repl) + start = -1 + } + if ret != nil { + ret = append(ret, c) + } + } + if start >= 0 && ret != nil { + ret = append(ret, repl) + } + if ret == nil { + return source + } + return ret +} + +// ToRune decode given bytes start at pos and returns a rune. +func ToRune(source []byte, pos int) rune { + i := pos + for ; i >= 0; i-- { + if utf8.RuneStart(source[i]) { + break + } + } + r, _ := utf8.DecodeRune(source[i:]) + return r +} + +// ToValidRune returns 0xFFFD if the given rune is invalid, otherwise v. +func ToValidRune(v rune) rune { + if v == 0 || !utf8.ValidRune(v) { + return rune(0xFFFD) + } + return v +} + +// ToLinkReference converts given bytes into a valid link reference string. +// ToLinkReference performs unicode case folding, trims leading and trailing spaces, converts into lower +// case and replace spaces with a single space character. +func ToLinkReference(v []byte) string { + v = TrimLeftSpace(v) + v = TrimRightSpace(v) + v = DoFullUnicodeCaseFolding(v) + return string(ReplaceSpaces(v, ' ')) +} + +var htmlEscapeTable = [256][]byte{nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, []byte("""), nil, nil, nil, []byte("&"), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, []byte("<"), nil, []byte(">"), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil} + +// EscapeHTMLByte returns HTML escaped bytes if the given byte should be escaped, +// otherwise nil. +func EscapeHTMLByte(b byte) []byte { + return htmlEscapeTable[b] +} + +// EscapeHTML escapes characters that should be escaped in HTML text. +func EscapeHTML(v []byte) []byte { + cob := NewCopyOnWriteBuffer(v) + n := 0 + for i := 0; i < len(v); i++ { + c := v[i] + escaped := htmlEscapeTable[c] + if escaped != nil { + cob.Write(v[n:i]) + cob.Write(escaped) + n = i + 1 + } + } + if cob.IsCopied() { + cob.Write(v[n:]) + } + return cob.Bytes() +} + +// UnescapePunctuations unescapes blackslash escaped punctuations. +func UnescapePunctuations(source []byte) []byte { + cob := NewCopyOnWriteBuffer(source) + limit := len(source) + n := 0 + for i := 0; i < limit; { + c := source[i] + if i < limit-1 && c == '\\' && IsPunct(source[i+1]) { + cob.Write(source[n:i]) + cob.WriteByte(source[i+1]) + i += 2 + n = i + continue + } + i++ + } + if cob.IsCopied() { + cob.Write(source[n:]) + } + return cob.Bytes() +} + +// ResolveNumericReferences resolve numeric references like 'Ӓ" . +func ResolveNumericReferences(source []byte) []byte { + cob := NewCopyOnWriteBuffer(source) + buf := make([]byte, 6, 6) + limit := len(source) + ok := false + n := 0 + for i := 0; i < limit; i++ { + if source[i] == '&' { + pos := i + next := i + 1 + if next < limit && source[next] == '#' { + nnext := next + 1 + if nnext < limit { + nc := source[nnext] + // code point like #x22; + if nnext < limit && nc == 'x' || nc == 'X' { + start := nnext + 1 + i, ok = ReadWhile(source, [2]int{start, limit}, IsHexDecimal) + if ok && i < limit && source[i] == ';' { + v, _ := strconv.ParseUint(BytesToReadOnlyString(source[start:i]), 16, 32) + cob.Write(source[n:pos]) + n = i + 1 + runeSize := utf8.EncodeRune(buf, ToValidRune(rune(v))) + cob.Write(buf[:runeSize]) + continue + } + // code point like #1234; + } else if nc >= '0' && nc <= '9' { + start := nnext + i, ok = ReadWhile(source, [2]int{start, limit}, IsNumeric) + if ok && i < limit && i-start < 8 && source[i] == ';' { + v, _ := strconv.ParseUint(BytesToReadOnlyString(source[start:i]), 0, 32) + cob.Write(source[n:pos]) + n = i + 1 + runeSize := utf8.EncodeRune(buf, ToValidRune(rune(v))) + cob.Write(buf[:runeSize]) + continue + } + } + } + } + i = next - 1 + } + } + if cob.IsCopied() { + cob.Write(source[n:]) + } + return cob.Bytes() +} + +// ResolveEntityNames resolve entity references like 'ö" . +func ResolveEntityNames(source []byte) []byte { + cob := NewCopyOnWriteBuffer(source) + limit := len(source) + ok := false + n := 0 + for i := 0; i < limit; i++ { + if source[i] == '&' { + pos := i + next := i + 1 + if !(next < limit && source[next] == '#') { + start := next + i, ok = ReadWhile(source, [2]int{start, limit}, IsAlphaNumeric) + if ok && i < limit && source[i] == ';' { + name := BytesToReadOnlyString(source[start:i]) + entity, ok := LookUpHTML5EntityByName(name) + if ok { + cob.Write(source[n:pos]) + n = i + 1 + cob.Write(entity.Characters) + continue + } + } + } + i = next - 1 + } + } + if cob.IsCopied() { + cob.Write(source[n:]) + } + return cob.Bytes() +} + +var htmlSpace = []byte("%20") + +// URLEscape escape the given URL. +// If resolveReference is set true: +// 1. unescape punctuations +// 2. resolve numeric references +// 3. resolve entity references +// +// URL encoded values (%xx) are kept as is. +func URLEscape(v []byte, resolveReference bool) []byte { + if resolveReference { + v = UnescapePunctuations(v) + v = ResolveNumericReferences(v) + v = ResolveEntityNames(v) + } + cob := NewCopyOnWriteBuffer(v) + limit := len(v) + n := 0 + + for i := 0; i < limit; { + c := v[i] + if urlEscapeTable[c] == 1 { + i++ + continue + } + if c == '%' && i+2 < limit && IsHexDecimal(v[i+1]) && IsHexDecimal(v[i+1]) { + i += 3 + continue + } + u8len := utf8lenTable[c] + if u8len == 99 { // invalid utf8 leading byte, skip it + i++ + continue + } + if c == ' ' { + cob.Write(v[n:i]) + cob.Write(htmlSpace) + i++ + n = i + continue + } + if int(u8len) > len(v) { + u8len = int8(len(v) - 1) + } + if u8len == 0 { + i++ + n = i + continue + } + cob.Write(v[n:i]) + stop := i + int(u8len) + if stop > len(v) { + i++ + n = i + continue + } + cob.Write(StringToReadOnlyBytes(url.QueryEscape(string(v[i:stop])))) + i += int(u8len) + n = i + } + if cob.IsCopied() && n < limit { + cob.Write(v[n:]) + } + return cob.Bytes() +} + +// FindURLIndex returns a stop index value if the given bytes seem an URL. +// This function is equivalent to [A-Za-z][A-Za-z0-9.+-]{1,31}:[^<>\x00-\x20]* . +func FindURLIndex(b []byte) int { + i := 0 + if !(len(b) > 0 && urlTable[b[i]]&7 == 7) { + return -1 + } + i++ + for ; i < len(b); i++ { + c := b[i] + if urlTable[c]&4 != 4 { + break + } + } + if i == 1 || i > 33 || i >= len(b) { + return -1 + } + if b[i] != ':' { + return -1 + } + i++ + for ; i < len(b); i++ { + c := b[i] + if urlTable[c]&1 != 1 { + break + } + } + return i +} + +var emailDomainRegexp = regexp.MustCompile(`^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*`) + +// FindEmailIndex returns a stop index value if the given bytes seem an email address. +func FindEmailIndex(b []byte) int { + // TODO: eliminate regexps + i := 0 + for ; i < len(b); i++ { + c := b[i] + if emailTable[c]&1 != 1 { + break + } + } + if i == 0 { + return -1 + } + if i >= len(b) || b[i] != '@' { + return -1 + } + i++ + if i >= len(b) { + return -1 + } + match := emailDomainRegexp.FindSubmatchIndex(b[i:]) + if match == nil { + return -1 + } + return i + match[1] +} + +var spaces = []byte(" \t\n\x0b\x0c\x0d") + +var spaceTable = [256]int8{0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + +var punctTable = [256]int8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + +// a-zA-Z0-9, ;/?:@&=+$,-_.!~*'()# +var urlEscapeTable = [256]int8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + +var utf8lenTable = [256]int8{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 99, 99, 99, 99, 99, 99, 99, 99} + +var urlTable = [256]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 5, 5, 1, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1, 1, 0, 1, 0, 1, 1, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 1, 1, 1, 1, 1, 1, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1} + +var emailTable = [256]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + +// UTF8Len returns a byte length of the utf-8 character. +func UTF8Len(b byte) int8 { + return utf8lenTable[b] +} + +// IsPunct returns true if the given character is a punctuation, otherwise false. +func IsPunct(c byte) bool { + return punctTable[c] == 1 +} + +// IsPunctRune returns true if the given rune is a punctuation, otherwise false. +func IsPunctRune(r rune) bool { + return int32(r) <= 256 && IsPunct(byte(r)) || unicode.IsPunct(r) +} + +// IsSpace returns true if the given character is a space, otherwise false. +func IsSpace(c byte) bool { + return spaceTable[c] == 1 +} + +// IsSpaceRune returns true if the given rune is a space, otherwise false. +func IsSpaceRune(r rune) bool { + return int32(r) <= 256 && IsSpace(byte(r)) || unicode.IsSpace(r) +} + +// IsNumeric returns true if the given character is a numeric, otherwise false. +func IsNumeric(c byte) bool { + return c >= '0' && c <= '9' +} + +// IsHexDecimal returns true if the given character is a hexdecimal, otherwise false. +func IsHexDecimal(c byte) bool { + return c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F' +} + +// IsAlphaNumeric returns true if the given character is a alphabet or a numeric, otherwise false. +func IsAlphaNumeric(c byte) bool { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' +} + +// A BufWriter is a subset of the bufio.Writer . +type BufWriter interface { + io.Writer + Available() int + Buffered() int + Flush() error + WriteByte(c byte) error + WriteRune(r rune) (size int, err error) + WriteString(s string) (int, error) +} + +// A PrioritizedValue struct holds pair of an arbitrary value and a priority. +type PrioritizedValue struct { + // Value is an arbitrary value that you want to prioritize. + Value interface{} + // Priority is a priority of the value. + Priority int +} + +// PrioritizedSlice is a slice of the PrioritizedValues +type PrioritizedSlice []PrioritizedValue + +// Sort sorts the PrioritizedSlice in ascending order. +func (s PrioritizedSlice) Sort() { + sort.Slice(s, func(i, j int) bool { + return s[i].Priority < s[j].Priority + }) +} + +// Remove removes the given value from this slice. +func (s PrioritizedSlice) Remove(v interface{}) PrioritizedSlice { + i := 0 + found := false + for ; i < len(s); i++ { + if s[i].Value == v { + found = true + break + } + } + if !found { + return s + } + return append(s[:i], s[i+1:]...) +} + +// Prioritized returns a new PrioritizedValue. +func Prioritized(v interface{}, priority int) PrioritizedValue { + return PrioritizedValue{v, priority} +} + +func bytesHash(b []byte) uint64 { + var hash uint64 = 5381 + for _, c := range b { + hash = ((hash << 5) + hash) + uint64(c) + } + return hash +} + +// BytesFilter is a efficient data structure for checking whether bytes exist or not. +// BytesFilter is thread-safe. +type BytesFilter interface { + // Add adds given bytes to this set. + Add([]byte) + + // Contains return true if this set contains given bytes, otherwise false. + Contains([]byte) bool + + // Extend copies this filter and adds given bytes to new filter. + Extend(...[]byte) BytesFilter +} + +type bytesFilter struct { + chars [256]uint8 + threshold int + slots [][][]byte +} + +// NewBytesFilter returns a new BytesFilter. +func NewBytesFilter(elements ...[]byte) BytesFilter { + s := &bytesFilter{ + threshold: 3, + slots: make([][][]byte, 64), + } + for _, element := range elements { + s.Add(element) + } + return s +} + +func (s *bytesFilter) Add(b []byte) { + l := len(b) + m := s.threshold + if l < s.threshold { + m = l + } + for i := 0; i < m; i++ { + s.chars[b[i]] |= 1 << uint8(i) + } + h := bytesHash(b) % uint64(len(s.slots)) + slot := s.slots[h] + if slot == nil { + slot = [][]byte{} + } + s.slots[h] = append(slot, b) +} + +func (s *bytesFilter) Extend(bs ...[]byte) BytesFilter { + newFilter := NewBytesFilter().(*bytesFilter) + newFilter.chars = s.chars + newFilter.threshold = s.threshold + for k, v := range s.slots { + newSlot := make([][]byte, len(v)) + copy(newSlot, v) + newFilter.slots[k] = v + } + for _, b := range bs { + newFilter.Add(b) + } + return newFilter +} + +func (s *bytesFilter) Contains(b []byte) bool { + l := len(b) + m := s.threshold + if l < s.threshold { + m = l + } + for i := 0; i < m; i++ { + if (s.chars[b[i]] & (1 << uint8(i))) == 0 { + return false + } + } + h := bytesHash(b) % uint64(len(s.slots)) + slot := s.slots[h] + if slot == nil || len(slot) == 0 { + return false + } + for _, element := range slot { + if bytes.Equal(element, b) { + return true + } + } + return false +} diff --git a/vendor/github.com/yuin/goldmark/util/util_safe.go b/vendor/github.com/yuin/goldmark/util/util_safe.go new file mode 100644 index 00000000..507a9d02 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/util/util_safe.go @@ -0,0 +1,13 @@ +// +build appengine js + +package util + +// BytesToReadOnlyString returns a string converted from given bytes. +func BytesToReadOnlyString(b []byte) string { + return string(b) +} + +// StringToReadOnlyBytes returns bytes converted from given string. +func StringToReadOnlyBytes(s string) []byte { + return []byte(s) +} diff --git a/vendor/github.com/yuin/goldmark/util/util_unsafe.go b/vendor/github.com/yuin/goldmark/util/util_unsafe.go new file mode 100644 index 00000000..d0988110 --- /dev/null +++ b/vendor/github.com/yuin/goldmark/util/util_unsafe.go @@ -0,0 +1,23 @@ +// +build !appengine,!js + +package util + +import ( + "reflect" + "unsafe" +) + +// BytesToReadOnlyString returns a string converted from given bytes. +func BytesToReadOnlyString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} + +// StringToReadOnlyBytes returns bytes converted from given string. +func StringToReadOnlyBytes(s string) (bs []byte) { + sh := (*reflect.StringHeader)(unsafe.Pointer(&s)) + bh := (*reflect.SliceHeader)(unsafe.Pointer(&bs)) + bh.Data = sh.Data + bh.Cap = sh.Len + bh.Len = sh.Len + return +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 118f8c1c..f7ddff62 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -229,6 +229,20 @@ github.com/stretchr/testify/assert # github.com/xanzy/ssh-agent v0.3.0 ## explicit github.com/xanzy/ssh-agent +# github.com/yuin/goldmark v1.4.4 +## explicit; go 1.16 +github.com/yuin/goldmark +github.com/yuin/goldmark/ast +github.com/yuin/goldmark/extension +github.com/yuin/goldmark/extension/ast +github.com/yuin/goldmark/parser +github.com/yuin/goldmark/renderer +github.com/yuin/goldmark/renderer/html +github.com/yuin/goldmark/text +github.com/yuin/goldmark/util +# github.com/yuin/goldmark-meta v1.0.0 +## explicit; go 1.15 +github.com/yuin/goldmark-meta # golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 ## explicit; go 1.11 golang.org/x/crypto/blowfish From 18da1bec2408fda9e16d351c7e6857a9a0d4682e Mon Sep 17 00:00:00 2001 From: gardener-robot-ci-3 <gardener.ci.user3@gmail.com> Date: Tue, 21 Dec 2021 10:41:16 +0000 Subject: [PATCH 2/6] Release v0.22.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index af33e750..0babdaf0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.22.0-dev \ No newline at end of file +v0.22.0 \ No newline at end of file From 97d637ba2f43e7dc7fdcfbeb71e417cea7ba0308 Mon Sep 17 00:00:00 2001 From: gardener-robot-ci-2 <gardener.ci.user2@gmail.com> Date: Tue, 21 Dec 2021 10:41:19 +0000 Subject: [PATCH 3/6] Prepare next Dev Cycle v0.23.0-dev --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0babdaf0..dfd14226 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.22.0 \ No newline at end of file +v0.23.0-dev \ No newline at end of file From 259a19c432511bbe61bdbeb7e433899ebcbe4c8f Mon Sep 17 00:00:00 2001 From: Dimitar Kostadinov <dimitar.kostadinov@gmail.com> Date: Tue, 4 Jan 2022 12:05:38 +0200 Subject: [PATCH 4/6] Fix Linkify regex & cleanup node tree --- pkg/markdown/link_modifier.go | 13 +- pkg/markdown/link_modifier_test.go | 4 +- pkg/markdown/parser.go | 6 +- .../git/git_resource_handler.go | 7 +- .../github/github_resource_handler.go | 6 +- pkg/resourcehandlers/github/github_utils.go | 2 +- pkg/util/files/file.go | 114 ------------- pkg/util/files/file_test.go | 151 ------------------ pkg/util/units/units.go | 13 -- 9 files changed, 23 insertions(+), 293 deletions(-) delete mode 100644 pkg/util/files/file.go delete mode 100644 pkg/util/files/file_test.go delete mode 100644 pkg/util/units/units.go diff --git a/pkg/markdown/link_modifier.go b/pkg/markdown/link_modifier.go index ae472127..19e0b49f 100644 --- a/pkg/markdown/link_modifier.go +++ b/pkg/markdown/link_modifier.go @@ -28,13 +28,13 @@ var ( }, } // defines an ordered list item marker without next not space char '[^ ]+' - marker = regexp.MustCompile(`^(\d{1,9}[.)]) {1,4}`) + marker = regexp.MustCompile(`^\d{1,9}[.)] {1,4}`) // defines a fence block line fence = regexp.MustCompile("^ {0,3}```") // GFM autolink extensions - http = regexp.MustCompile(`^https*://([a-zA-Z\d\-_]+\.)*[a-zA-Z\d\-]+\.[a-zA-Z\d\-]+[^ <]*$`) - www = regexp.MustCompile(`^www\.([a-zA-Z\d\-_]+\.)*[a-zA-Z\d\-]+\.[a-zA-Z\d\-]+[^ <]*$`) - email = regexp.MustCompile(`^[a-zA-Z\d.\-+]+@([a-zA-Z\d\-_]+\.)+[a-zA-Z\d\-_]+$`) + http = regexp.MustCompile(`^https?://(?:[a-zA-Z\d\-_]+\.)*[a-zA-Z\d\-]+\.[a-zA-Z\d\-]+[^ <]*$`) + www = regexp.MustCompile(`^www\.(?:[a-zA-Z\d\-_]+\.)*[a-zA-Z\d\-]+\.[a-zA-Z\d\-]+[^ <]*$`) + email = regexp.MustCompile(`^[a-zA-Z\d.\-+]+@(?:[a-zA-Z\d\-_]+\.)+[a-zA-Z\d\-_]+$`) ) // ResolveLink type defines function for modifying link destination @@ -869,6 +869,11 @@ func isClassicAutolink(link []byte, cnt []byte) bool { return true } if http.Match(link) { + if len(cnt) > 0 && cnt[len(cnt)-1] == '(' { + // workaround for using default Linkify URL regex in Hugo + // `(https://foo.bar).` will be rendered as classic autolink `(<https://foo.bar>).` + return true + } return false // GFM autolink http/s } if www.Match(link) { diff --git a/pkg/markdown/link_modifier_test.go b/pkg/markdown/link_modifier_test.go index 42873e93..3802037d 100644 --- a/pkg/markdown/link_modifier_test.go +++ b/pkg/markdown/link_modifier_test.go @@ -57,8 +57,8 @@ var _ = Describe("Links modifier", func() { Context("URL autolink", func() { BeforeEach(func() { lr.dst = "https://fake.com" - md = "links:\nhttp://foo.bar.baz\n<irc://foo.bar:2233/baz>\n<https://foo.bar.baz/test?q=hello&id=22&boolean~>\n(www.google.com/search?q=Markup+(business))\n" - exp = "links:\nhttps://fake.com\n<https://fake.com>\n<https://fake.com>\n(https://fake.com)\n" + md = "links:\nhttp://foo.bar.baz\n<irc://foo.bar:2233/baz>\n<https://foo.bar.baz/test?q=hello&id=22&boolean~>\n(www.google.com/search?q=Markup+(business))\n(https://foo.bar).\n" + exp = "links:\nhttps://fake.com\n<https://fake.com>\n<https://fake.com>\n(https://fake.com)\n(<https://fake.com>).\n" }) It("modifies links", func() { Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/markdown/parser.go b/pkg/markdown/parser.go index a215c40d..d5668195 100644 --- a/pkg/markdown/parser.go +++ b/pkg/markdown/parser.go @@ -11,16 +11,18 @@ import ( "github.com/yuin/goldmark/extension" "github.com/yuin/goldmark/parser" "github.com/yuin/goldmark/text" + "regexp" ) var ( + // extends Linkify regex by excluding trailing whitespaces and punctuations `[^\s<?!.,:*_~]` + urlRgx = regexp.MustCompile(`^(?:http|https|ftp)://[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-z]+(?::\d+)?(?:[/#?][-a-zA-Z0-9@:%_+.~#$!?&/=\(\);,'">\^{}\[\]` + "`" + `]*)?[^\s<?!.,:*_~]`) // parser extension for GitHub Flavored Markdown & Frontmatter support extensions = []goldmark.Extender{ extension.GFM, meta.Meta, } - // goldmark.Markdown parser with GFM extensions - gmParser = goldmark.New(goldmark.WithExtensions(extensions...)) + gmParser = goldmark.New(goldmark.WithExtensions(extensions...), goldmark.WithParserOptions(extension.WithLinkifyURLRegexp(urlRgx))) ) // Parse markdown content and returns AST node or error diff --git a/pkg/resourcehandlers/git/git_resource_handler.go b/pkg/resourcehandlers/git/git_resource_handler.go index ce11084e..4d29a086 100644 --- a/pkg/resourcehandlers/git/git_resource_handler.go +++ b/pkg/resourcehandlers/git/git_resource_handler.go @@ -190,13 +190,12 @@ func (g *Git) ResolveNodeSelector(ctx context.Context, node *api.Node, excludePa _node.SetParentsDownwards() // finally, cleanup folder entries from contentSelectors - for _, child := range _node.Nodes { - github.CleanupNodeTree(child) - } + github.CleanupNodeTree(_node) + if len(_node.Nodes) > 0 { return _node.Nodes, nil } - return nil, nil + return []*api.Node{}, nil } func (g *Git) getNodeSelectorLocalPath(repositoryPath string, rl *github.ResourceLocator) string { diff --git a/pkg/resourcehandlers/github/github_resource_handler.go b/pkg/resourcehandlers/github/github_resource_handler.go index c2202f96..19859389 100644 --- a/pkg/resourcehandlers/github/github_resource_handler.go +++ b/pkg/resourcehandlers/github/github_resource_handler.go @@ -305,9 +305,11 @@ func (gh *GitHub) ResolveNodeSelector(ctx context.Context, node *api.Node, exclu return nil, err } // finally, cleanup folder entries from contentSelectors - for _, child := range childNodes { - CleanupNodeTree(child) + _node := &api.Node{ + Nodes: childNodes, } + CleanupNodeTree(_node) + childNodes = _node.Nodes if childNodes == nil { return []*api.Node{}, nil } diff --git a/pkg/resourcehandlers/github/github_utils.go b/pkg/resourcehandlers/github/github_utils.go index 45e9f1c4..6267ae08 100644 --- a/pkg/resourcehandlers/github/github_utils.go +++ b/pkg/resourcehandlers/github/github_utils.go @@ -162,7 +162,7 @@ func isRawURL(u *url.URL) bool { // CleanupNodeTree cleanups node tree: // - remove contentSources that reference tree objects. They are used // internally to build the structure but are not a valid contentSource -// - remove empty nodes that do not contain markdown. The build algorithm +// - remove empty nodes that do not contain markdowns. The build algorithm // is blind for the content of a node and leaves nodes that are folders // containing for example images only and thus irrelevant to the // documentation structure diff --git a/pkg/util/files/file.go b/pkg/util/files/file.go deleted file mode 100644 index 1e4ef58b..00000000 --- a/pkg/util/files/file.go +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package files - -import ( - "fmt" - "path/filepath" - "time" - - "github.com/golang/glog" - "github.com/howeyc/fsnotify" -) - -const ( - watchDebounceDelay = 100 * time.Millisecond -) - -// Watcher encapsulates file watch and configuration, -// abstracting form the underlying file watch provider -type Watcher struct { - Watcher *fsnotify.Watcher - WatchedFiles []string - watched []string -} - -// NewFileWatcher creates Watcher -func NewFileWatcher() *Watcher { - return &Watcher{ - WatchedFiles: []string{}, - watched: []string{}, - } -} - -// AddToWatch adds files to a WatchedFiles list monitored -// by this webhook's Watcher. The Watcher is created on -// demand if it's nil when the operation is invoked. -func (w *Watcher) AddToWatch(files ...string) error { - if len(files) == 0 { - return nil - } - if w.Watcher == nil { - var err error - w.Watcher, err = fsnotify.NewWatcher() - if err != nil { - return err - } - } - w.WatchedFiles = append(w.WatchedFiles, files...) - return nil -} - -// Watch starts monitoring WatchedFiles, until a signal is received on its stop channel. If WatchedFiles -// or Watcher are not initialized, Watch returns immediately. The eventHandler function is invoked upon -// Modify/Create file events raised by the watched files. -func (w *Watcher) Watch(stop <-chan struct{}, eventHandler func() error) error { - var started bool - if w.Watcher == nil || w.WatchedFiles == nil || len(w.WatchedFiles) == 0 { - return nil - } - defer func(started bool) { - w.Watcher.Close() // nolint: errcheck - if started { - glog.V(6).Infof("watching files stopped") - } - }(started) - - // watch the parent directory of the target files so we can catch - // symlink updates of k8s ConfigMaps volumes. - for _, file := range w.WatchedFiles { - watchDir, _ := filepath.Split(file) - exists := false - for _, w := range w.watched { - if w == watchDir { - exists = true - break - } - } - if !exists { - if err := w.Watcher.Watch(watchDir); err != nil { - return fmt.Errorf("could not watch %v: %v", file, err) - } - glog.V(6).Infof("watching %s", watchDir) - w.watched = append(w.watched, watchDir) - } - } - started = true - glog.V(6).Info("watching files started") - var timerC <-chan time.Time - for { - select { - case <-timerC: - { - if eventHandler != nil { - if err := eventHandler(); err != nil { - break - } - } - } - case event := <-w.Watcher.Event: - { - // use a timer to debounce configuration updates - if event.IsModify() || event.IsCreate() { - timerC = time.After(watchDebounceDelay) - } - } - case err := <-w.Watcher.Error: - glog.V(10).Infof("watcher error: %v", err) - case <-stop: - return nil - } - } -} diff --git a/pkg/util/files/file_test.go b/pkg/util/files/file_test.go deleted file mode 100644 index a25ac3ae..00000000 --- a/pkg/util/files/file_test.go +++ /dev/null @@ -1,151 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package files - -import ( - "context" - "fmt" - "github.com/gardener/docforge/pkg/util/tests" - "github.com/howeyc/fsnotify" - "github.com/stretchr/testify/assert" - "math/rand" - "os" - "path/filepath" - "testing" - "time" -) - -const ( - // test data directory - testdata = "testdata" -) - -func init() { - tests.SetKlogV(10) -} - -func runFileModify(ctx context.Context, t *testing.T, filePath string, period time.Duration) { - ticker := time.NewTicker(period) - defer ticker.Stop() - for { - select { - case <-ticker.C: - { - var f *os.File - f, err := os.OpenFile(filePath, os.O_WRONLY, 0600) - if err != nil { - t.Errorf("%v", err) - } - fi, _ := f.Stat() - mtime := fmt.Sprintf("%v", fi.ModTime().UTC()) - if _, err = f.WriteString("test " + mtime); err != nil { - t.Errorf("%v", err) - } - f.Close() - } - case <-ctx.Done(): - return - } - } -} - -func TestAddToWatch(t *testing.T) { - actual := NewFileWatcher() - - err := actual.AddToWatch("/etc/watched0", "/etc/watched1") - - assert.Nil(t, err) - assert.Equal(t, []string{"/etc/watched0", "/etc/watched1"}, actual.WatchedFiles) -} - -func TestWatchDebounce(t *testing.T) { - watcher, err := fsnotify.NewWatcher() - if err != nil { - t.Errorf("%v", err) - return - } - watchedDirPath := filepath.Join(testdata, fmt.Sprintf("dir%d", rand.Intn(1<<32))) - if err := os.MkdirAll(watchedDirPath, os.ModePerm); err != nil { - t.Errorf("%v", err) - return - } - defer func() { - if err := os.RemoveAll(watchedDirPath); err != nil { - t.Errorf("%v", err) - } - watcher.Close() - }() - watchedFilePath := filepath.Join(watchedDirPath, fmt.Sprintf("watched%d", rand.Intn(1<<32))) - var f *os.File - if f, err = os.Create(watchedFilePath); err != nil { - t.Errorf("%v", err) - return - } - f.Close() - wh := &Watcher{ - WatchedFiles: []string{watchedFilePath}, - Watcher: watcher, - } - ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) - defer cancel() - - invocations := 0 - go func() { - err = wh.Watch(ctx.Done(), func() error { - invocations++ - return nil - }) - }() - // trigger high-rate Modify events and stop before the watch duration - debounce period - // to give debouncer a chance to invoke the func after its debounce period - go func() { - ctx, cancel := context.WithTimeout(context.Background(), 350*time.Millisecond) - defer cancel() - runFileModify(ctx, t, watchedFilePath, 50*time.Millisecond) - <-ctx.Done() - }() - - <-ctx.Done() - - assert.Equal(t, 1, invocations) -} - -func TestWatchNoWatcher(t *testing.T) { - wh := &Watcher{ - WatchedFiles: []string{"watched"}, - } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) - defer cancel() - invocations := 0 - err := wh.Watch(ctx.Done(), func() error { - invocations++ - return nil - }) - - assert.Nil(t, err) - assert.Equal(t, 0, invocations) -} - -func TestWatchNoWatchedFiles(t *testing.T) { - watcher, err := fsnotify.NewWatcher() - defer watcher.Close() - wh := &Watcher{ - WatchedFiles: []string{}, - Watcher: watcher, - } - - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer cancel() - - invocations := 0 - err = wh.Watch(ctx.Done(), func() error { - invocations++ - return nil - }) - - assert.Nil(t, err) - assert.Equal(t, 0, invocations) -} diff --git a/pkg/util/units/units.go b/pkg/util/units/units.go deleted file mode 100644 index 2712ebcf..00000000 --- a/pkg/util/units/units.go +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-FileCopyrightText: 2020 SAP SE or an SAP affiliate company and Gardener contributors -// -// SPDX-License-Identifier: Apache-2.0 - -package units - -// Byte size suffixes. -const ( - B int64 = 1 - KB int64 = 1 << (10 * iota) - MB - GB -) From 9d44486313889bf21600338084c708f0c186307f Mon Sep 17 00:00:00 2001 From: gardener-robot-ci-3 <gardener.ci.user3@gmail.com> Date: Wed, 5 Jan 2022 08:36:28 +0000 Subject: [PATCH 5/6] Release v0.23.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index dfd14226..964a04d9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.23.0-dev \ No newline at end of file +v0.23.0 \ No newline at end of file From aa69fb46cf1304b64028ff309b07aa45396fb2e9 Mon Sep 17 00:00:00 2001 From: gardener-robot-ci-3 <gardener.ci.user3@gmail.com> Date: Wed, 5 Jan 2022 08:36:32 +0000 Subject: [PATCH 6/6] Prepare next Dev Cycle v0.24.0-dev --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 964a04d9..a262dd75 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.23.0 \ No newline at end of file +v0.24.0-dev \ No newline at end of file