Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added transform command. #29

Merged
merged 13 commits into from
May 19, 2021
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Changed to transform.
Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>
  • Loading branch information
bwplotka committed May 17, 2021
commit 600e9bf547239147de17d3433daea692edf71aea
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@ require (
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
github.com/efficientgo/tools v0.0.0-20201227131601-4dde9fba2562
github.com/go-kit/kit v0.10.0
github.com/gobwas/glob v0.2.3
github.com/gocolly/colly/v2 v2.1.0
github.com/gohugoio/hugo v0.74.3
github.com/mattn/go-shellwords v1.0.10
14 changes: 7 additions & 7 deletions main.go
Original file line number Diff line number Diff line change
@@ -17,8 +17,8 @@ import (
"github.com/bwplotka/mdox/pkg/mdformatter"
"github.com/bwplotka/mdox/pkg/mdformatter/linktransformer"
"github.com/bwplotka/mdox/pkg/mdformatter/mdgen"
"github.com/bwplotka/mdox/pkg/transform"
"github.com/bwplotka/mdox/pkg/version"
"github.com/bwplotka/mdox/pkg/web"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/oklog/run"
@@ -67,7 +67,7 @@ func main() {

ctx, cancel := context.WithCancel(context.Background())
registerFmt(ctx, app)
registerWeb(ctx, app)
registerTransform(ctx, app)

cmd, runner := app.Parse()
logger := setupLogger(*logLevel, *logFormat)
@@ -203,11 +203,11 @@ func validateAnchorDir(anchorDir string, files []string) (_ string, err error) {
return anchorDir, nil
}

func registerWeb(_ context.Context, app *extkingpin.App) {
cmd := app.Command("web", "Pre process markdown files to allow it for use for popular static HTML websites based on markdown source code and front matter options")
cfg := cmd.Flag("config", "Path to the YAML file with spec defined in github.com/bwplotka/mdox/pkg/web.Config").
Short('c').Default(".mdox.web.yaml").ExistingFile()
func registerTransform(_ context.Context, app *extkingpin.App) {
cmd := app.Command("transform", "Transform markdown files in various ways. For example pre process markdown files to allow it for use for popular static HTML websites based on markdown source code and front matter options.")
cfg := cmd.Flag("config", "Path to the YAML file with spec defined in github.com/bwplotka/mdox/pkg/transform.Config").
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe a section can be added to README about transform command, with sample config yaml, and that can be linked instead of struct? 🙂

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, we could use mdox-exec with our tool we plan to build and do this at the point? =D Great idea!

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure!

Short('c').Default(".mdox.yaml").ExistingFile()
cmd.Run(func(ctx context.Context, logger log.Logger) error {
return web.ProcessGitHubDir(ctx, logger, *cfg)
return transform.Dir(ctx, logger, *cfg)
})
}
18 changes: 11 additions & 7 deletions pkg/mdformatter/mdformatter.go
Original file line number Diff line number Diff line change
@@ -71,7 +71,7 @@ func WithLinkTransformer(l LinkTransformer) Option {
}
}

// WithMetaBlockTransformer allows you to override the default CodeBlockTransformer.
// WithCodeBlockTransformer allows you to override the default CodeBlockTransformer.
func WithCodeBlockTransformer(cb CodeBlockTransformer) Option {
return func(m *Formatter) {
m.cb = cb
@@ -81,7 +81,7 @@ func WithCodeBlockTransformer(cb CodeBlockTransformer) Option {
func New(ctx context.Context, opts ...Option) *Formatter {
f := &Formatter{
ctx: ctx,
fm: FormatFrontMatter{},
fm: FormatFrontMatterTransformer{},
}
for _, opt := range opts {
opt(f)
@@ -97,11 +97,15 @@ func (RemoveFrontMatter) TransformFrontMatter(_ SourceContext, _ map[string]inte

func (RemoveFrontMatter) Close() error { return nil }

type FormatFrontMatter struct{}
type FormatFrontMatterTransformer struct{}

func (FormatFrontMatter) TransformFrontMatter(_ SourceContext, frontMatter map[string]interface{}) ([]byte, error) {
func (FormatFrontMatterTransformer) TransformFrontMatter(_ SourceContext, frontMatter map[string]interface{}) ([]byte, error) {
return FormatFrontMatter(frontMatter), nil
}

func FormatFrontMatter(frontMatter map[string]interface{}) []byte {
if len(frontMatter) == 0 {
return nil, nil
return nil
}

keys := make([]string, 0, len(frontMatter))
@@ -133,10 +137,10 @@ func (FormatFrontMatter) TransformFrontMatter(_ SourceContext, frontMatter map[s
_, _ = fmt.Fprintf(b, "\n%v: %v", k, frontMatter[k])
}
_, _ = b.Write([]byte("\n---\n\n"))
return b.Bytes(), nil
return b.Bytes()
}

func (FormatFrontMatter) Close(SourceContext) error { return nil }
func (FormatFrontMatterTransformer) Close(SourceContext) error { return nil }

type Diffs []gitdiff.Diff

104 changes: 104 additions & 0 deletions pkg/transform/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
package transform

import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"text/template"

"github.com/gobwas/glob"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
)

type Config struct {
Version int
InputDir string `yaml:"inputDir"`
OutputDir string `yaml:"outputDir"`

Transformations []*TransformationConfig

// GitIgnore specifies if output dir should be git ignored or not.
GitIgnore bool `yaml:"GitIgnore"`
}

type TransformationConfig struct {
_glob glob.Glob

// Glob matches files using https://github.com/gobwas/glob.
// After first match, file is no longer matching other elements.
Glob string

// Skip skips moving matched files.
Skip bool

// Path is an optional different path for the file to be moved.
// NOTE: All relative links will be moved accordingly.
Path string

// FrontMatter holds front matter transformations.
FrontMatter *FrontMatterConfig `yaml:"frontMatter"`
}

type FrontMatterConfig struct {
_template *template.Template

// Template represents Go template that will be rendered as font matter.
// This will override any existing font matter.1
// TODO(bwplotka): Add add only option?
Template string
}

func parseConfigFile(configFile string) (Config, error) {
configFile, err := filepath.Abs(configFile)
if err != nil {
return Config{}, errors.Wrap(err, "abs")
}
c, err := ioutil.ReadFile(configFile)
if err != nil {
return Config{}, errors.Wrap(err, "read config file")
}
return ParseConfig(c)
}

func ParseConfig(c []byte) (Config, error) {
cfg := Config{}
if err := yaml.Unmarshal(c, &cfg); err != nil {
return Config{}, errors.Wrapf(err, "parsing template content %q", string(c))
}

if cfg.InputDir == "" {
return Config{}, errors.New("contentDir field is required")
}

d, err := os.Stat(cfg.InputDir)
if err != nil {
return Config{}, err
}
if !d.IsDir() {
return Config{}, errors.New("contentDir field is not pointing directory")
}
cfg.InputDir = strings.TrimSuffix(cfg.InputDir, "/")

if cfg.OutputDir == "" {
return Config{}, errors.New("outputDir field is required")
}
cfg.OutputDir = strings.TrimSuffix(cfg.OutputDir, "/")

for _, f := range cfg.Transformations {
f._glob, err = glob.Compile(f.Glob, '/')
if err != nil {
return Config{}, errors.Wrapf(err, "compiling glob %v", f.Glob)
}

if f.FrontMatter != nil {
f.FrontMatter._template, err = template.New("").Parse(f.FrontMatter.Template)
if err != nil {
return Config{}, errors.Wrapf(err, "compiling template %v", f.FrontMatter.Template)
}
}
}

return cfg, nil
}
1 change: 1 addition & 0 deletions pkg/transform/testdata/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
tmp/*
23 changes: 23 additions & 0 deletions pkg/transform/testdata/mdox1.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
version: 1

inputDir: "testdata/testproj"
outputDir: "testdata/tmp/1"

gitIgnored: true

transformations:
- glob: "/README.md"
path: _index.md
frontMatter:
template: |
title: "{{ .FirstHeader }}"

cascade:
- type: "docs"
_target:
path: "/**"

- glob: "**/README.md"
frontMatter:
template: |
title: "{{ .FirstHeader }}"
36 changes: 36 additions & 0 deletions pkg/transform/testdata/mdox2.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
version: 1

inputDir: "testdata/testproj"
outputDir: "testdata/tmp/2"

gitIgnored: true

transformations:
- glob: "/README.md"
path: _index.md
frontMatter:
template: |
title: "{{ .FirstHeader }}"

cascade:
- type: "docs"
_target:
path: "/**"

- glob: "/doc.md"
frontMatter:
template: |
title: "{{ .FirstHeader }}"

- glob: "/Team/doc.md"
path: inner/doc.md
frontMatter:
template: |
title: "{{ .FirstHeader }}"
yolo: "yolo"

- glob: "**/README.md"
path: /test1/_index.md
frontMatter:
template: |
title: "{{ .FirstHeader }}"
5 changes: 5 additions & 0 deletions pkg/transform/testdata/testproj/Proposals/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Proposals

[RelLink](../../../tmp/2/test1/_index.md)

[RelLink](../Team/doc.md)
7 changes: 7 additions & 0 deletions pkg/transform/testdata/testproj/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Group Handbook

Yolo

[RelLink](Proposals/README.md)

[RelLink](Team/doc.md)
7 changes: 7 additions & 0 deletions pkg/transform/testdata/testproj/Team/doc.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Some Doc

[RelLink](#some-doc)

[RelLink](../README.md#group-handbook)

[RelLink](../Proposals/README.md)
Loading