diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml new file mode 100644 index 00000000..906b3f46 --- /dev/null +++ b/.github/workflows/deps.yml @@ -0,0 +1,21 @@ +# https://github.com/actions/go-dependency-submission +name: dependency-submission +on: + push: + branches: + - main + +permissions: + contents: write + +jobs: + main: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version: "1.21.x" + - uses: actions/go-dependency-submission@v2 + with: + go-mod-path: go.mod diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..2b4ff7b4 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,63 @@ +name: main + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + go-version: ["1.21.x"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go-version }} + - run: go test -race -v -timeout 10s -short ./... + + builds: + strategy: + fail-fast: true + matrix: + go-version: ["1.21.x"] + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version: stable + - name: Build fabric + run: go build -v -o ${{ runner.temp }}/fabric + + golangci-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version: "1.21.x" + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.55.2 + + markdownlint-cli: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: nosborn/github-action-markdown-cli@v3.3.0 + with: + files: docs/ + config_file: .markdownlint.yaml + dot: true diff --git a/.github/workflows/snyk-security.yml b/.github/workflows/snyk-security.yml deleted file mode 100644 index 7411825e..00000000 --- a/.github/workflows/snyk-security.yml +++ /dev/null @@ -1,82 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -# A sample workflow which sets up Snyk to analyze the full Snyk platform (Snyk Open Source, Snyk Code, -# Snyk Container and Snyk Infrastructure as Code) -# The setup installs the Snyk CLI - for more details on the possible commands -# check https://docs.snyk.io/snyk-cli/cli-reference -# The results of Snyk Code are then uploaded to GitHub Security Code Scanning -# -# In order to use the Snyk Action you will need to have a Snyk API token. -# More details in https://github.com/snyk/actions#getting-your-snyk-token -# or you can signup for free at https://snyk.io/login -# -# For more examples, including how to limit scans to only high-severity issues -# and fail PR checks, see https://github.com/snyk/actions/ - -name: Snyk Security - -on: - workflow_dispatch: - -# on: -# push: -# branches: [$default-branch, $protected-branches] -# pull_request: -# branches: [$default-branch] - -permissions: - contents: read - -jobs: - snyk: - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Snyk CLI to check for security issues - # Snyk can be used to break the build when it detects security issues. - # In this case we want to upload the SAST issues to GitHub Code Scanning - uses: snyk/actions/setup@806182742461562b67788a64410098c9d9b96adb - - # For Snyk Open Source you must first set up the development environment for your application's dependencies - # For example for Node - #- uses: actions/setup-node@v3 - # with: - # node-version: 16 - - env: - # This is where you will need to introduce the Snyk API token created with your Snyk account - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - - # Runs Snyk Code (SAST) analysis and uploads result into GitHub. - # Use || true to not fail the pipeline - - name: Snyk Code test - run: snyk code test --sarif > snyk-code.sarif # || true - - # Runs Snyk Open Source (SCA) analysis and uploads result to Snyk. - - name: Snyk Open Source monitor - run: snyk monitor --all-projects - - # Runs Snyk Infrastructure as Code (IaC) analysis and uploads result to Snyk. - # Use || true to not fail the pipeline. - - name: Snyk IaC test and report - run: snyk iac test --report # || true - - # Build the docker image for testing - - name: Build a Docker image - run: docker build -t your/image-to-test . - # Runs Snyk Container (Container and SCA) analysis and uploads result to Snyk. - - name: Snyk Container monitor - run: snyk container monitor your/image-to-test --file=Dockerfile - - # Push the Snyk Code results into GitHub Code Scanning tab - - name: Upload result to GitHub Code Scanning - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: snyk-code.sarif diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 83ab9c1e..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,43 +0,0 @@ -on: - workflow_dispatch: - # push: - # branches: - # - main - # pull_request: - -permissions: - contents: read - -name: Test -jobs: - matrix-test: - name: test - strategy: - matrix: - go-version: ['1.21.x'] - os: [ubuntu-latest, macos-latest, windows-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - - name: Build fabric - run: go build -v -o ${{ runner.temp }}/fabric - - name: Build fabric-plugins - run: go build -v -o ${{ runner.temp }}/plugins ./cmd/plugins - - name: Test - run: go test -race -v -timeout 10s ./... - - golangci: - name: lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version: '1.21.x' - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - with: - version: v1.55.2 \ No newline at end of file diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 00000000..57a15d51 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,12 @@ +# https://github.com/DavidAnson/markdownlint/blob/main/schema/.markdownlint.yaml +default: true # enable all by default +MD007: # unordered list indentation + indent: 2 +MD013: false # do not validate line length +MD014: false # allow $ before command output +MD024: false # allow duplicate headings, for example, "Example" +MD029: # ordered list prefix, use all ones + style: "one" +MD033: false # allow inline HTML +MD041: false # the first line is not always H1 +MD025: false # because it thinks that is there is frontmatter, there should be no H1 diff --git a/README.md b/README.md index a601c58e..4c6bd571 100644 --- a/README.md +++ b/README.md @@ -14,44 +14,35 @@ Codifying and automating mission-critical communications with standardized and r -
- fabric-diagram -
- - - > [!NOTE] > Fabric is currently in the early stages of development, and there may be some issues. We welcome your feedback, so if you have any suggestions, ideas, or encounter bugs, please share them with us in our [Slack](https://fabric-community.slack.com/). Fabric is an open-source configuration language and a CLI tool that enables the codification and automation of the content generation process. -[Fabric diagram TBD] +
+ fabric-diagram +
Fabric produces rich Markdown documents from the template files that declaratively define data requirements and content structure. The templates consist of reusable blocks powered by plugins. Fabric fetches data from various external data stores, security solutions, and platforms using a collection of integrations and renders the Markdown document through a set of content blocks. Since the output file is just a Markdown document, it can be quickly rendered into PDF / HTML using any 3rd party renderer. - # Installation [TBD] - # Usage [TBD] - # Documentation Visit [https://blackstork.io/fabric/docs/](https://blackstork.io/fabric/docs/) for full documentation. - # Security Please report any suspected security vulnerabilities through GitHub's [security advisory reporting](https://github.com/blackstork-io/fabric/security/advisories/new). We take all legitimate reports seriously and will thoroughly investigate. We kindly request that you talk to us before making any public disclosures. This ensures that no excessive information is revealed before a patch is ready and users have sufficient time to upgrade. - # License Fabric is licensed under Apache-2.0 license. See the [LICENSE](LICENSE) file for the details. diff --git a/docs/_index.md b/docs/_index.md index 6336a438..5fcf3fd8 100644 --- a/docs/_index.md +++ b/docs/_index.md @@ -1,5 +1,5 @@ --- -title: Overview +title: What is Fabric? menuName: Docs type: docs menus: main @@ -10,7 +10,6 @@ weight: 1 Fabric is an open-source configuration language and a CLI tool that enables the codification and automation of content generation process. The language includes components like documents, data definitions and content blocks. - ## Get Started - [Install]({{< ref "install.md" >}}) — how to install and run Fabric. diff --git a/docs/language/_index.md b/docs/language/_index.md index 6746df37..46c3f6c8 100644 --- a/docs/language/_index.md +++ b/docs/language/_index.md @@ -4,8 +4,7 @@ type: docs weight: 2 --- - -# Fabric Configuration Language +# Fabric Configuration Language Fabric Configuration Language (FCL) serves as the core feature for Fabric — a powerful tool designed to streamline document generation. FCL enables users to express data requirements and template structures within configuration files with lightweight syntax. @@ -48,10 +47,8 @@ config data elasticsearch { } ``` - See [Syntax](./syntax/) for more details on the FCL syntax. - ## IDE Support Since Fabric configuration language is based on HCL, IDE extensions for HCL syntax highlighting will work with Fabric files out of the box. @@ -59,6 +56,3 @@ Since Fabric configuration language is based on HCL, IDE extensions for HCL synt If you are using Microsoft VSCode, take a look at [Fabric Extension for Visual Studio Code](https://github.com/blackstork-io/vscode-fabric). ![VSCode Fabric Extension screenshot](./vscode-fabric-screenshot.png) - - - diff --git a/docs/language/configs.md b/docs/language/configs.md index b0d0bcc2..4fedad34 100644 --- a/docs/language/configs.md +++ b/docs/language/configs.md @@ -18,43 +18,41 @@ fabric { There can be only one `fabric` block defined within the codebase. - ### Supported Arguments - `plugin_versions`: (required) a map that matches namespaced plugin names to the version constraints in SemVer (see Terraform's [version constraint syntax](https://developer.hashicorp.com/terraform/language/expressions/version-constraints#version-constraint-syntax)). - `cache_dir`: (optional) a path to a directory on the local file system. The default value is `./.fabric`. If the directory does not exist, it will be created on the first run of `fabric`. - ### Supported Nested Blocks - `plugin_registry` – (optional) a block that defines available plugin registries. The block accepts only one attribute + ```hcl plugin_registry { cache_dir = "/tmp/plugins/" } ``` - - `mirror_dir` – (optional) a path to a directory on the local filesystem with plugin archives. + - `mirror_dir` – (optional) a path to a directory on the local filesystem with plugin archives. ### Example ```hcl fabric { - plugins_registry { - mirror_dir = "/tmp/plugins/" - } + plugins_registry { + mirror_dir = "/tmp/plugins/" + } - cache_dir = "./.fabric" + cache_dir = "./.fabric" - plugin_versions = { - "blackstork/data.elasticsearch" = "1.2.3" - "blackstork/content.openai" = "=11.22.33" - } + plugin_versions = { + "blackstork/data.elasticsearch" = "1.2.3" + "blackstork/content.openai" = "=11.22.33" + } } ``` - ## Plugin Configuration `config` block defines a configuration for a plugin: @@ -76,12 +74,10 @@ If `` is not provided, the block is treated as a default configurat If `` is set, the config block can be explicitely referenced inside the `content` or `data` block. This is helpful if there is a need to have multiple configurations for the same plugin. - ### Supported Arguments The arguments that are allowed in the configuration block are plugin-specific – every plugin defines the configuration options supported. See [Plugins]({{< ref "plugins.md" >}}) for the details on the plugin configuation paramters. - ### Supported Nested Blocks Nested blocks are not supported inside the `config` blocks. @@ -100,17 +96,16 @@ data csv "events_a" { document "test-document" { - data ref { - base = data.csv.events_a - } + data ref { + base = data.csv.events_a + } - data csv "events_b" { - config { - delimiter = ","; - } + data csv "events_b" { + config { + delimiter = ","; + } - path = "/tmp/events-b.csv" - } + path = "/tmp/events-b.csv" + } } ``` - diff --git a/docs/language/content-blocks.md b/docs/language/content-blocks.md index 5c7476c9..569a5fce 100644 --- a/docs/language/content-blocks.md +++ b/docs/language/content-blocks.md @@ -29,18 +29,16 @@ document "foobar" { } ``` -If a `content` block is defined at the root level of the configuration file, outside of the `document`, both names – the plugin name and the block name – must be provided. A combination of block type `content`, plugin name and block name serves as a unique identifier of a block within the codebase. +If a `content` block is defined at the root level of the configuration file, outside of the `document`, both names – the plugin name and the block name – must be provided. A combination of block type `content`, plugin name and block name serves as a unique identifier of a block within the codebase. If a `content` block is defined within the document, only plugin name is required, while block name is optional. A content block is implemented by a content plugin specified by `` in a block signature. See [Plugins]({{< ref "plugins.md" >}}) for the details on the plugins supported by Fabric. - ## Supported Arguments The arguments provided in the block are either generic arguments or plugin-specific input parameters. - ### Generic Arguments - `config`: (optional) a reference to a named config block defined outside the document. If provided, it takes precedence over the default configuration for the plugin. See [Plugin Configuration]({{< ref "configs.md#plugin-configuration" >}}) for the details. @@ -50,18 +48,15 @@ The arguments provided in the block are either generic arguments or plugin-speci - `query_input_required`: (optional) a boolean flag that specifies if `query_input` must be explicitely provided when the content block is referenced. `false` by default. See [Query Input Requirement]({{< ref "references.md#query-input-requirement" >}}) for the details. - `query_input`: (optional) a JQ query to be applied to the context object. The results of the query are stored under `query_input` field in the context. See [Query Input Requirement]({{< ref "references.md#query-input-requirement" >}}) for the details. - ### Plugin-specific Arguments Plugin-specific arguments are defined by a plugin specification. See [Plugins]({{< ref "plugins.md" >}}) for the details on the supported arguments per plugin. - ## Supported Nested Blocks - `meta`: (optional) a block containing metadata for the block. - `config`: (optional) an inline configuration for the block. If provided, it takes precedence over the `config` argument and default configuration for the plugin. - ## Context When Fabric renders a content block, a corresponding content plugin is called. Along with [plugin configuration]({{< ref "configs/#plugin-configuration" >}}) and input parameters, a plugin receives the context object with all data available. @@ -71,12 +66,10 @@ The context object is a JSON dictionary with pre-defined root-level fields: - `data` contains a map of all resolved data definitions for the document. The JSON path to a specific data point follows the data block signature: `data..`. - `query_result` contains a result of the execution of a JQ query provided in `query` attribute, as requested by `query_input_required` attribute. This is mostly used in `ref` blocks to explicitely provide data to the query in `query` attribute. - ## References See [References]({{< ref references.md >}}) for the details about referencing content blocks. - ## Example ```hcl diff --git a/docs/language/data-blocks.md b/docs/language/data-blocks.md index 96dcecf6..7d0a9388 100644 --- a/docs/language/data-blocks.md +++ b/docs/language/data-blocks.md @@ -26,34 +26,27 @@ The plugin name and the block name are required, and, together with the block ty The data, represented by the block, is accessible under `data..` path in the context (see [Context]({{< ref "content-blocks.md#context" >}}) for more details). - ## Supported Arguments The arguments provided in the block are either generic arguments or plugin-specific input parameters. - ### Generic Arguments - `config`: (optional) a reference to a named config block defined outside the document. If provided, it takes precedence over the default configuration for the plugin. See [Plugin Configuration]({{< ref "configs.md#plugin-configuration" >}}) for the details. - ### Plugin-specific Arguments Plugin-specific arguments are defined by a plugin specification. See [Plugins]({{< ref "plugins.md" >}}) for the details on the supported arguments per plugin. - ## Supported Nested Blocks - `meta`: (optional) a block containing metadata for the block. - `config`: (optional) an inline configuration for the block. If provided, it takes precedence over the `config` argument and default configuration for the plugin. - - ## References See [References]({{< ref references.md >}}) for the details about referencing data blocks. - ## Example ```hcl diff --git a/docs/language/documents.md b/docs/language/documents.md index a69e5e9c..d03b96d8 100644 --- a/docs/language/documents.md +++ b/docs/language/documents.md @@ -23,11 +23,9 @@ The `document` block is a shell that groups the data definitions, the section, a - `title`: (optional) a title of the document. It is a syntax sugar for a nested `content` block that renders a title. The title content block precedes any other nested `content` blocks or `sequence` blocks defined at the same level. - ## Supported Nested Blocks - `meta`: (optional) a block containing metadata for the block. - `data`: see [Data Blocks]({{< ref data-blocks.md >}}) for the details. - `content`: see [Content Blocks]({{< ref content-blocks.md >}}) for the details. - `section`: see [Section Blocks]({{< ref section-blocks.md >}}) for the details. - diff --git a/docs/language/references.md b/docs/language/references.md index 74584b6a..c09fa4b6 100644 --- a/docs/language/references.md +++ b/docs/language/references.md @@ -12,93 +12,84 @@ To include a block defined outside the document, on the root level of a file, in ```hcl "" { - ... + ... } document "foo" { - ref "" { - base = - - ... - } + ref "" { + base = + ... + } } ``` A block identifier is a dot-separated list of block type, plugin name and a block name: + ```hcl .. ``` If the `ref` block is defined at the root level of the file, outside of the `document` definition, the block name is required. If the `ref` block is defined within the document, the block name is optional. - {{< hint warning >}} An anonymous `ref` block adops a name of the referenced block. Since the final name of the block is not stated explicitely, unwanted overrides can happen — a block signature must be unique between the blocks defined on the same level. {{< /hint >}} - ## Argument Overrides The `ref` block definition can include the attribute that would override the attributes provided in the original block. This is very helpful if the block's behaviour needs to be adjusted per document. - ## Query Input Requirement -Content blocks rely on `query` attribute for selecting data needed for rendering (see content blocks' [Generic Arguments]({{< ref "content-blocks.md#generic-arguments" >}})). The JQ query uses the data path which is often document-specific and depends on the name of the data block. This hinders the reusability of the content blocks. +Content blocks rely on `query` attribute for selecting data needed for rendering (see content blocks' [Generic Arguments]({{< ref "content-blocks.md#generic-arguments" >}})). The JQ query uses the data path which is often document-specific and depends on the name of the data block. This hinders the reusability of the content blocks. Fabric supports an explicit way for the content block to require the input data - `query_input` and `query_input_required` attributes. If `query_input_required` is set to `true`, the content block expectes `query_input` attribute to be provided in the `ref` block. - ## Example ```hcl data elasticsearch "foo" { - index = "test-index" - ... + index = "test-index" + ... } content text "qux" { - # Using `query_input` field in the context that contains the result of - # the `query_input` query - query = ".query_input | length" + # Using `query_input` field in the context that contains the result of + # the `query_input` query + query = ".query_input | length" - # Require the referrer to specify `query_input` query that will be used - # to get the data for `query_input` field in the context - query_input_required = true + # Require the referrer to specify `query_input` query that will be used + # to get the data for `query_input` field in the context + query_input_required = true - text = "The data contains {{ .query_result }} elements" + text = "The data contains {{ .query_result }} elements" } - document "test-document" { - # Anonymous referrer block adops the name of the referenced block - `data.elasticsearch.foo` - data ref { - base = data.elasticsearch.foo - } - - # Named referrer block keeps its name - `data.elasticsearch.bar` - data ref "bar" { - base = data.elasticsearch.foo - } - - # Provided argument `index` overrides the value set in the original block. - data ref "baz" { - base = data.elasticsearch.foo - index = "another-test-index" - } - - # Referred block requires `query_input` to be provided, - # so it can be used in query set in `query` argument in the original block. - content ref { - base = content.text.qux - query_input = ".data.elasticsearch.bar" - } + # Anonymous referrer block adops the name of the referenced block - `data.elasticsearch.foo` + data ref { + base = data.elasticsearch.foo + } + + # Named referrer block keeps its name - `data.elasticsearch.bar` + data ref "bar" { + base = data.elasticsearch.foo + } + + # Provided argument `index` overrides the value set in the original block. + data ref "baz" { + base = data.elasticsearch.foo + index = "another-test-index" + } + + # Referred block requires `query_input` to be provided, + # so it can be used in query set in `query` argument in the original block. + content ref { + base = content.text.qux + query_input = ".data.elasticsearch.bar" + } } ``` - - - - diff --git a/docs/language/section-blocks.md b/docs/language/section-blocks.md index 5a87cf08..c3bb8145 100644 --- a/docs/language/section-blocks.md +++ b/docs/language/section-blocks.md @@ -34,19 +34,16 @@ If a `section` block is defined at the root level of the configuration file, out If a `section` block is defined within the document template, the section name is optional. - ## Supported Arguments - `title`: (optional) represents the title of the content group. It is a syntactic sugar for a `content` block that renders a title. The title content block takes precedence over any other nested `content` blocks or `section` blocks defined at the same level. - ## Supported Nested Blocks - `meta`: (optional) a block containing metadata for the block. - `content`: see [Content Blocks]({{< ref content-blocks.md >}}) for the details. - `section`: a block of type `section` can be embedded into another `section` block. - ## References See [References]({{< ref references.md >}}) for the details about referencing section blocks. diff --git a/docs/language/syntax.md b/docs/language/syntax.md index 185f3218..bba03ccf 100644 --- a/docs/language/syntax.md +++ b/docs/language/syntax.md @@ -10,7 +10,6 @@ This page describes native syntax of the Fabric Configuration Language (FCL). FC Fabric language syntax has two core components: arguments and blocks. - ## Arguments The arguments are used for assigning a value to a name inside a block: @@ -23,7 +22,6 @@ The arguments are used for assigning a value to a name inside a block: The argument name (`query_string` in the snippet above) can contain letters, digits, underscores (`_`), and hyphens (`-`). The first character of an identifier must not be a digit. - ## Blocks A block is a container that defines a configuration, a data requirement or a content structure. @@ -49,7 +47,6 @@ Supported categories of blocks: - [Content definitions]({{< ref "content-blocks.md" >}}): `content` block - [Content structure]({{< ref "section-blocks.md" >}}): `section` block - ## Comments Fabric language supports three different comment syntaxes: @@ -60,6 +57,6 @@ Fabric language supports three different comment syntaxes: We recommend to use `#` single-line comment style in most cases. Future automatic code formatting tools will prioritise `#` comments as idiomatic. - ## Character Encoding + Fabric configuration files must be UTF-8 encoded. Fabric allows non-ASCII characters in comments, and string values. diff --git a/internal/elasticsearch/data_elasticsearch_test.go b/internal/elasticsearch/data_elasticsearch_test.go index 57b372ee..aecccb16 100644 --- a/internal/elasticsearch/data_elasticsearch_test.go +++ b/internal/elasticsearch/data_elasticsearch_test.go @@ -38,7 +38,7 @@ func TestElasticSearchDataSuite(t *testing.T) { func (s *ElasticSearchDataTestSuite) SetupSuite() { s.ctx = context.Background() opts := []testcontainers.ContainerCustomizer{ - testcontainers.WithImage("docker.elastic.co/elasticsearch/elasticsearch:8.9.0"), + testcontainers.WithImage("docker.io/elasticsearch:8.9.0"), elasticsearch.WithPassword("password123"), } container, err := elasticsearch.RunContainer(s.ctx, opts...)