diff --git a/CHANGELOG.md b/CHANGELOG.md index 8180daef7c..e85b30c32a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## v0.7 ++ Refactor gateways #147 ++ Fixed sensor error state recovery bug #145 ++ Ability to add annotations on sensor and gateway pods #143 ++ Github gateway ++ Added support for NATS standard and streaming as communication protocol between gateway + and sensor #99 + ## v0.6 + Gitlab Gateway #120 + If sensor is repeatable then deploy it as deployment else job #109 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8950e3e586..a9284ce3d8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,7 @@ make all Follow the rest of the [quickstart guide](docs/quickstart.md) to install components. ## Changing Types -If you're making a change to the `pkg/apis/sensor/v1alpha1` package, please ensure you re-run the K8 code-generator scripts found in the `/hack` folder. First, ensure you have the `generate-groups.sh` script at the path: `vendor/k8s.io/code-generator/`. Next run the following commands in order: +If you're making a change to the `pkg/apis` package, please ensure you re-run the K8 code-generator scripts found in the `/hack` folder. First, ensure you have the `generate-groups.sh` script at the path: `vendor/k8s.io/code-generator/`. Next run the following commands in order: ``` $ make codegen ``` diff --git a/Makefile b/Makefile index 8757678e9d..ecc0955b10 100644 --- a/Makefile +++ b/Makefile @@ -194,33 +194,33 @@ mqtt-image: mqtt-linux # Custom gateways storage-grid: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/storagegrid-gateway ./gateways/custom/storagegrid/cmd + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/storagegrid-gateway ./gateways/community/storagegrid/cmd storage-grid-linux: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make storage-grid storage-grid-image: storage-grid-linux - docker build -t $(IMAGE_PREFIX)storage-grid-gateway:$(IMAGE_TAG) -f ./gateways/custom/storagegrid/Dockerfile . + docker build -t $(IMAGE_PREFIX)storage-grid-gateway:$(IMAGE_TAG) -f ./gateways/community/storagegrid/Dockerfile . @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)storage-grid-gateway:$(IMAGE_TAG) ; fi gitlab: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/gitlab-gateway ./gateways/custom/gitlab/cmd + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/gitlab-gateway ./gateways/community/gitlab/cmd gitlab-linux: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make gitlab gitlab-image: gitlab-linux - docker build -t $(IMAGE_PREFIX)gitlab-gateway:$(IMAGE_TAG) -f ./gateways/custom/gitlab/Dockerfile . + docker build -t $(IMAGE_PREFIX)gitlab-gateway:$(IMAGE_TAG) -f ./gateways/community/gitlab/Dockerfile . @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)gitlab-gateway:$(IMAGE_TAG) ; fi github: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/github-gateway ./gateways/custom/github/cmd + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/github-gateway ./gateways/community/github/cmd github-linux: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make github github-image: github-linux - docker build -t $(IMAGE_PREFIX)github-gateway:$(IMAGE_TAG) -f ./gateways/custom/github/Dockerfile . + docker build -t $(IMAGE_PREFIX)github-gateway:$(IMAGE_TAG) -f ./gateways/community/github/Dockerfile . @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)github-gateway:$(IMAGE_TAG) ; fi test: diff --git a/README.md b/README.md index b0b3359af8..979bfd834b 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,14 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/argoproj/argo-events)](https://goreportcard.com/report/github.com/argoproj/argo-events) ## What is Argo Events? -Argo Events is an open source event-based dependency manager for Kubernetes. The core concept of the project are - * `gateways` which are implemented as a Kubernetes-native Custom Resource Definition that either produce the events internally or process the events that originate from outside the gateways +Argo Events is an event-based dependency manager for Kubernetes. The core concept of the project are + * `gateways` which are implemented as a Kubernetes-native Custom Resource Definition that process events from event source. - * `sensors` which are implemented as a Kubernetes-native Custom Resource Definition that define a set of dependencies and trigger actions. + * `sensors` which are implemented as a Kubernetes-native Custom Resource Definition that define a set of event dependencies and trigger actions. - - Define multiple dependencies from a variety of gateway sources + - Define multiple dependencies from a variety of gateway event sources - Build custom gateways to support business-level constraint logic - - Trigger messages and Kubernetes object creation after successful dependency resolution + - Trigger Kubernetes object creation after successful dependency resolution - Trigger escalation after errors, or dependency constraint failures - Build and manage a distributed, cross-team, event-driven architecture - Easily leverage Kubernetes-native APIs to monitor dependencies @@ -23,10 +23,36 @@ Argo Events is an open source event-based dependency manager for Kubernetes. The - Scalable & Resilient. - Simple or Complex dependencies. Manage everything from simple, linear, real-time dependencies to complex, multi-source, batch job dependencies. +
+
+ +![](docs/architecture.png) + +
+
+ ## Getting Started [![asciicast](https://asciinema.org/a/207973.png)](https://asciinema.org/a/207973) -- [Installation](./docs/quickstart.md) -- [Guide](./docs/tutorial.md) -- [Want to contribute or develop/run locally?](./CONTRIBUTING.md) -- See where the project is headed in the [roadmap](./ROADMAP.md) +
+
+ +1. [Installation](./docs/quickstart.md) +2. [Sensor and gateway controllers](docs/controllers-guide.md) +3. [Learn about gateways](docs/gateway-guide.md) +4. [Learn about sensors](docs/sensor-guide.md) +5. [Learn about triggers](docs/trigger-guide.md) +6. Install Gateways and Sensors + 1. [Webhook](gateways/core/webhook/install.md) + 2. [Artifact](gateways/core/artifact/install.md) + 3. [Calendar](gateways/core/calendar/install.md) + 4. [Resource](gateways/core/resource/install.md) + 5. [File](gateways/core/file/install.md) + 6. Streams + 1. [NATS](gateways/core/stream/nats/install.md) + 2. [KAFKA](gateways/core/stream/kafka/install.md) + 3. [AMQP](gateways/core/stream/amqp/install.md) + 4. [MQTT](gateways/core/stream/mqtt/install.md) +7. [Write your own gateway](docs/custom-gateway.md) +8. [Want to contribute or develop/run locally?](./CONTRIBUTING.md) +9. See where the project is headed in the [roadmap](./ROADMAP.md) diff --git a/ROADMAP.md b/ROADMAP.md index 70c08b84fd..192022a392 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,9 +1,8 @@ # Roadmap ## Development Phase +- Boolean logic for event dependencies in sensor - E2E testing -- Add NATS Streaming as internal pub-sub system -- Add SNS & SQS gateway Support ## Idea Phase - Use [hashicorp/go-getter](https://github.com/hashicorp/go-getter) for the `ArtifactReader` interface. diff --git a/docs/architecture.png b/docs/architecture.png index a5515f4c8f..f2de1b75e0 100644 Binary files a/docs/architecture.png and b/docs/architecture.png differ diff --git a/docs/artifact-guide.md b/docs/artifact-guide.md index b88645a4f8..b4ffc84495 100644 --- a/docs/artifact-guide.md +++ b/docs/artifact-guide.md @@ -6,60 +6,14 @@ Inlined artifacts are included directly within the sensor resource and decoded a ## S3 Amazon Simple Storage Service (S3) is a block/file/object store for the internet. The standardized [API](https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html) allows storage and retrieval of data at any time from anywhere on the web. There are a number of S3 providers that include, but are not limited to: -* [Amazon S3](https://aws.amazon.com/s3/?nc2=h_m1) -* [Minio](https://minio.io/) -* [NetApp](https://www.netapp.com/us/products/data-management-software/object-storage-grid-sds.aspx) -* [CEPH](http://docs.ceph.com/docs/master/radosgw/s3/) -* [Rook](https://rook.io/) - ### Minio -Argo Events uses the [minio-go](https://github.com/minio/minio-go) client for access to any Amazon S3 compatible object store. [Minio](https://www.minio.io/) is an distributed object storage server. Follow the Minio [Bucket Notification Guide](https://docs.minio.io/docs/minio-bucket-notification-guide) for help with configuring your minio server to listen and monitor for bucket event notifications. Note that you will need to setup a supported message queue for configuring your notification targets (i.e. NATS, WebHooks, Kafka, etc.). - -#### Installation on Kubernetes -The [Minio Deployment Quickstart Guide](https://docs.minio.io/docs/minio-deployment-quickstart-guide.html) is useful for help in getting Minio up & running on your orchestration platform. We've also outlined additional steps below to use Minio for signal notifications and as an object store for trigger resources. - -1. Install the Helm chart -``` -$ helm init -... -$ helm install stable/minio --name artifacts --set service.type=LoadBalancer -... - -$ #Verify that the minio pod, the minio service and minio secret are present -$ kubectl get all -n default -l app=minio - -NAME READY STATUS RESTARTS AGE -pod/artifacts-minio-85547b6bd9-bhtt8 1/1 Running 0 21m - -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -service/artifacts-minio ClusterIP None 9000/TCP 21m - -NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE -deployment.apps/artifacts-minio 1 1 1 1 21m - -NAME DESIRED CURRENT READY AGE -replicaset.apps/artifacts-minio-85547b6bd9 1 1 1 21m -``` - -2. Create a bucket in Minio and upload the hello-world.yaml into that bucket. -Download the hello-world.yaml from https://raw.githubusercontent.com/argoproj/argo/master/examples/hello-world.yaml -``` -$ kubectl port-forward `kubectl get pod -l app=minio -o name` 9000:9000 -``` -Open the browser at http://localhost:9000 -Create a new bucket called 'workflows'. -Upload the hello-world.yaml into that bucket - +Argo Events uses the [minio-go](https://github.com/minio/minio-go) client for access to any Amazon S3 compatible object store. [Minio](https://www.minio.io/) is an distributed object storage server. Follow the Minio [Name Notification Guide](https://docs.minio.io/docs/minio-bucket-notification-guide) for help with configuring your minio server to listen and monitor for bucket event notifications. Note that you will need to setup a supported message queue for configuring your notification targets (i.e. NATS, WebHooks, Kafka, etc.). -#### Enabling bucket notifications -Once the Minio server is configured with a notification target and you have restarted the server to put the changes into effect, you now need to explicitely enable event notifications for a specified bucket. Enabling these notifications are out of scope of Argo Events since bucket notifications are a construct within Minio that exists at the `bucket` level. To avoid multiple sensors on the same S3 bucket conflicting with each other, creating, updating, and deleting Minio bucket notifications should be delegated to a separate process with knowledge of all notification targets including those outside of the Argo Events. -``` -$ k edit configmap artifacts-minio -$ k delete pod artifacts-minio -``` +## File +Artifacts are defined in a file that is mounted via a [PersistentVolume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) within the `sensor-controller` pod. -## File (future enhancement) -This will enable access to file artifacts via a filesystem mounted as a [PersistentVolume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) within the `sensor-controller` pod. +## URL +Artifacts are accessed from web via RESTful API. -## URL (future enhancement) -This will enable access to web artifacts via RESTful API. \ No newline at end of file +## Configmap +Artifact stored in Kubernetes configmap are accessed using the key. \ No newline at end of file diff --git a/docs/controllers-guide.md b/docs/controllers-guide.md index 1cba6a96b3..c5cc5161c8 100644 --- a/docs/controllers-guide.md +++ b/docs/controllers-guide.md @@ -1,6 +1,6 @@ ## Controllers -* Sensor and Gateway controller are the components which manage Sensor and Gateway resources. +* Sensor and Gateway controllers are the components which manage Sensor and Gateway resources. * Sensor and Gateway are Kubernetes Custom Resources. For more information on K8 CRDs visit, https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/ @@ -20,9 +20,9 @@ data: namespace: my-custom-namespace # optional ``` -If you don't provide namespace, controller will watch all namespaces for gateway resource. +`namespace`: If you don't provide namespace, controller will watch all namespaces for gateway resource. -Note on `instance-id`: it is used to map a gateway or sensor to a controller. +`instance-id`: it is used to map a gateway or sensor to a controller. e.g. when you create a gateway with label `gateways.argoproj.io/gateway-controller-instanceid: argo-events`, a controller with label `argo-events` will process that gateway. `instance-id` for controller are managed using [controller-configmap](https://raw.githubusercontent.com/argoproj/argo-events/master/hack/k8s/manifests/gateway-controller-configmap.yaml) Basically `instance-id` is used to horizontally scale controllers, so you won't end up overwhelming a controller with large @@ -32,73 +32,6 @@ Basically `instance-id` is used to horizontally scale controllers, so you won't ### Gateway controller Gateway controller watches gateway resource and manages lifecycle of a gateway. -```yaml -# The gateway-controller listens for changes on the gateway CRD and creates gateway -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: gateway-controller -spec: - replicas: 1 - template: - metadata: - labels: - app: gateway-controller - spec: - serviceAccountName: argo-events-sa - containers: - - name: gateway-controller - image: argoproj/gateway-controller:latest - imagePullPolicy: Always - env: - - name: GATEWAY_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: GATEWAY_CONTROLLER_CONFIG_MAP - value: gateway-controller-configmap -``` - -* Lifecycle of Gateway in gateway controller - - ![](gateway-controller-fsm.png) ### Sensor controller Sensor controller watches sensor resource and manages lifecycle of a sensor. -```yaml -# The sensor-controller listens for changes on the sensor CRD and creates sensor executor jobs -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: sensor-controller -spec: - replicas: 1 - template: - metadata: - labels: - app: sensor-controller - spec: - serviceAccountName: argo-events-sa - containers: - - name: sensor-controller - image: argoproj/sensor-controller:latest - imagePullPolicy: Always - env: - - name: SENSOR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: SENSOR_CONFIG_MAP - value: sensor-controller-configmap -``` - -* Lifecycle of a Sensor managed by sensor controller - - ![](sensor-fsm.png) - -
- -* Lifecycle of Sensor within sensor pod - - ![](sensor-pod-fsm.png) - \ No newline at end of file diff --git a/docs/core-gateway-style.png b/docs/core-gateway-style.png deleted file mode 100644 index 4dc9c56283..0000000000 Binary files a/docs/core-gateway-style.png and /dev/null differ diff --git a/docs/custom-gateway.md b/docs/custom-gateway.md index 77814636fc..6bbeb2e4e8 100644 --- a/docs/custom-gateway.md +++ b/docs/custom-gateway.md @@ -1,161 +1,43 @@ -# Custom Gateway +# Implement a Custom Gateway -You can write a gateway in 4 different ways, +To implement a custom gateway, you need to create a gRPC server and implement the service defined below. +The framework code acts as a gRPC client consuming event stream from gateway server. -1. Core gateway style. -2. Implement gateway as a gRPC server. -3. Implement gateway as a HTTP server. -4. Write your own implementation completely independent of framework. +1. [Proto definition](#proto-definition) +2. [Available environment variables](#available-environment-variables-to-server) +3. [Implementation](#implementation) -Difference between first three options and fourth is that in options 1,2 and 3, framework provides a mechanism -to watch configuration updates, start/stop a configuration dynamically. In option 4, its up to -user to watch configuration updates and take actions. - Below are the environment variables provided to all types of gateways +## Proto Definition +1. The proto file is located at https://github.com/argoproj/argo-events/blob/master/gateways/eventing.proto + +2. If you choose to implement the gateway in `go`, then you can find generated client stubs at https://github.com/argoproj/argo-events/blob/master/gateways/eventing.pb.go + +3. To create stubs in other languages, head over to gRPC website https://grpc.io/ + +4. Service + ```proto + /** + * Service for handling event sources. + */ + service Eventing { + // StartEventSource starts an event source and returns stream of events. + rpc StartEventSource(EventSource) returns (stream Event); + // ValidateEventSource validates an event source. + rpc ValidateEventSource(EventSource) returns (ValidEventSource); + } + ``` + +## Available Environment Variables to Server | Field | Description | |----------------------|--------------| - | TRANSFORMER_PORT | http server port running within gateway transformer | - | GATEWAY_NAMESPACE | namespace of the gateway | - | GATEWAY_PROCESSOR_CONFIG_MAP | map containing configurations to run in a gateway| - | GATEWAY_NAME | name of the gateway | + | GATEWAY_NAMESPACE | K8s namespace of the gateway | + | GATEWAY_EVENT_SOURCE_CONFIG_MAP | K8s configmap containing event source| + | GATEWAY_NAME | name of the gateway | | GATEWAY_CONTROLLER_INSTANCE_ID | gateway controller instance id | + | GATEWAY_CONTROLLER_NAME | gateway controller name + | GATEWAY_SERVER_PORT | Port on which the gateway gRPC server should run -## Core Gateway Style -The most straightforward option. The gateway consists of two components, - -1. Gateway Processor: either generates events internally or listens for external events and then -passes those events to gateway-transformer - -2. Gateway Transformer: transforms incoming events into cloudevents specification compliant events -and dispatches them to watchers. - -![](core-gateway-style.png) - -User needs to implement following interface. - -```go -type ConfigExecutor interface { - StartConfig(configContext *ConfigContext) - StopConfig(configContext *ConfigContext) - Validate(configContext *ConfigContext) error -} -``` - -ConfigData contains configuration key/name, value and the state of the configuration -```go -type ConfigData struct { - // Data holds the actual configuration - Data *ConfigData - - StartChan chan struct{} - - StopChan chan struct{} - - DataChan chan []byte - - DoneChan chan struct{} - - ErrChan chan error - - ShutdownChan chan struct{} - - // Active tracks configuration state as running or stopped - Active bool - // Cancel is called to cancel the context used by client to communicate with gRPC server. - // Use it only if gateway is implemented as gRPC server. - Cancel context.CancelFunc -} -``` - -GatewayConfig contains generic configuration for a gateway -```go -type GatewayConfig struct { - // Log provides fast and simple logger dedicated to JSON output - Log zerolog.Logger - // Clientset is client for kubernetes API - Clientset kubernetes.Interface - // Name is gateway name - Name string - // Namespace is namespace for the gateway to run inside - Namespace string - // KubeConfig rest client config - KubeConfig *rest.Config -} -``` - -* To send events back to framework for further processing, use -```go -gatewayConfig.DispatchEvent(event []byte, src string) error -``` - -For detailed implementation, check out [Core Gateways](https://github.com/argoproj/argo-events/tree/master/gateways/core) - -## gRPC gateway -A gRPC gateway has 3 components, -1. Gateway Processor Server - your implementation of gRPC streaming server, either generates events or listens to -external events and streams them back to gateway-processor-client - -2. Gateway Processor Client - gRPC client provided by framework that connects to gateway processor server. - -3. Gateway Transformer: transforms incoming events into cloudevents specification compliant events - and dispatches them to interested watchers. - -### Architecture - ![](grpc-gateway.png) - -To implement gateway processor server, you will need to implement -```proto -RunGateway(GatewayConfig) returns (stream Event) -``` -`RunGateway` method takes a gateway configuration and sends events over a stream. - -The gateway processor client opens a new connection for each gateway configuration and starts listening to -events on a stream. - -For detailed implementation, check out [Calendar gRPC gateway](https://github.com/argoproj/argo-events/tree/master/gateways/grpc/calendar) - -* To run gRPC gateway, you need to provide `rpcPort` in gateway spec. - -* To write gateway gRPC server using other languages than go, generate server interfaces using protoc. -Follow protobuf tutorials []()https://developers.google.com/protocol-buffers/docs/tutorials - -## HTTP Gateway -A gRPC gateway has 3 components, -1. Gateway Processor Server - your implementation of HTTP streaming server, either generates events or listens to -external events and streams them back to gateway-processor-client. User code must accept POST requests on `/start` and `/stop` -endpoints. - -2. Gateway Processor Client - sends configuration to gateway-processor-server on either `/start` endpoint for -a new configuration or `/stop` endpoint to stop a configuration. Processor client itself has a HTTP server -running internally listening for events from gateway-processor-server. - -3. Gateway Transformer: transforms incoming events into cloudevents specification compliant events - and dispatches them to watchers. - - -### Architecture -![](http-gateway.png) - -List of environment variables available to user code - -| Field | Description | -|----------------------|--------------| -| GATEWAY_PROCESSOR_SERVER_HTTP_PORT | Gateway processor server HTTP server port | -| GATEWAY_PROCESSOR_CLIENT_HTTP_PORT | Gateway processor client HTTP server port | -| GATEWAY_HTTP_CONFIG_START | Endpoint to post new configuration | -| GATEWAY_HTTP_CONFIG_STOP | Endpoint to make configuration to stop | -| GATEWAY_HTTP_CONFIG_EVENT | Endpoint to send events to | -| GATEWAY_HTTP_CONFIG_RUNNING | Endpoint to send activation notifications for the configuration / -| GATEWAY_HTTP_CONFIG_ERROR | Endpoint on which which gateway processor listens for errors from configurations | - -For detailed implementation, check out [Calendar HTTP gateway](https://github.com/argoproj/argo-events/tree/master/gateways/rest/calendar) - -## Framework independent -The fourth option is you provide gateway implementation from scratch: watch the configuration -updates, start/stop configuration if needed. Only requirement is that events must be -dispatched to gateway-transformer using HTTP post request. The port to dispatch the event -is made available through environment variable `TRANSFORMER_PORT`. - -### Gateway Examples -* Example gateway definitions are available at [here](https://github.com/argoproj/argo-events/tree/master/examples/gateways) \ No newline at end of file +## Implementation +You can find existing gateway implementation at https://github.com/argoproj/argo-events/tree/master/gateways/core \ No newline at end of file diff --git a/docs/gateway-controller-fsm.png b/docs/gateway-controller-fsm.png deleted file mode 100644 index fe8537f0dd..0000000000 Binary files a/docs/gateway-controller-fsm.png and /dev/null differ diff --git a/docs/gateway-guide.md b/docs/gateway-guide.md index 2b392ba41e..8a930a37e0 100644 --- a/docs/gateway-guide.md +++ b/docs/gateway-guide.md @@ -1,206 +1,71 @@ # Gateway Guide +1. [What is a gateway](#what-is-a-gateway) +2. [Components](#components) +2. [Specification](#specification) +4. [Managing Event Sources](#managing-event-sources) +5. [How to write a custom gateway?](#how-to-write-a-custom-gateway) +6. [Examples](#examples) + ## What is a gateway? -A gateway is a long running/repeatable process whose tasks are to process and transform either the internally produced events or -external events into the [cloudevents specification](https://github.com/cloudevents/spec) compliant events and dispatch them to watchers(sensors and/or gateways). +A gateway is a long running/repeatable process whcih consumes events from event sources, transform them into the [cloudevents specification](https://github.com/cloudevents/spec) compliant events and dispatch them to watchers(sensors and/or gateways). -## Gateway Components +### Components A gateway has two components: - 1. gateway-processor: Either generates the events internally or listens to external events. - The implementation of gateway-processor is provided by the user which means the user can easily create a custom gateway. - - 2. gateway-transformer: Transforms the incoming events from gateway-processor into a cloudevents specification compliant events. - The event is then dispatched to watchers. + 1. gateway-client: It creates one or more gRPC clients depending on event sources, consumes events from server, transforms these events into cloudevents and dispatches them to watchers. + Refer https://github.com/cloudevents/spec for more info on cloudevents specifications. + + 2. gateway-server: It is a gRPC server that consumes event from event sources and dispatches to gRPC client/s created by gateway client - Refer https://github.com/cloudevents/spec for more info on cloudevents specifications. +### Core gateways + 1. **Calendar**: + Events produced can be based on a [cron](https://crontab.guru/) schedule or an [interval duration](https://golang.org/pkg/time/#ParseDuration). In addition, calendar gateway currently supports a `recurrence` field in which to specify special exclusion dates for which this gateway will not produce an event. -Core gateways come in 5 types: + 2. **Webhooks**: + Webhook gateway expose a basic HTTP server endpoint/s. + Users can register multiple REST API endpoint. See Request Methods in RFC7231 to define the HTTP REST endpoint. -| Type | Description | -|-------------|-----------------------------------------| -| `Stream` | Listens to messages on a queue/topic | -| `Artifact` | Listens to S3 Bucket Notifications | -| `Calendar` | Produces events internally depending upon date/time schedules or intervals | -| `Resource` | Watches Kubernetes resources | -| `Webhook` | Reacts to HTTP webhook notifications (Git, JIRA, Trello etc.) | + 3. **Kubernetes Resources**: + Resource gateway support watching Kubernetes resources. Users can specify `group`, `version`, `kind`, and filters including prefix of the object name, labels, annotations, and createdBy time. + 4. **Artifacts**: + Artifact gateway support S3 `bucket-notifications` via [Minio](https://docs.minio.io/docs/minio-bucket-notification-guide). Note that a supported notification target must be running, exposed, and configured in the Minio server. For more information, please refer to the [artifact guide](artifact-guide.md). -\ -In order to take advantage of the various gateway types, you may need to install compatible message platforms (e.g. amqp, mmqp, NATS, etc..) and s3 api compatible object storage servers (e.g. Minio, Rook, CEPH, NetApp). See the [artifact guide](artifact-guide.md) for installing object stores. + 5. **Streams**: + Stream gateways contain a generic specification for messages received on a queue and/or though messaging server. The following are the `builtin` supported stream gateways. -## Architecture -![](architecture.png) + 1. **NATS**: + [Nats](https://nats.io/) is an open-sourced, lightweight, secure, and scalable messaging system for cloud native applications and microservices architecture. It is currently a hosted CNCF Project. - -## Gateway Controller -The `gateway-controller` is responsible for managing the `Gateway` resources. + 2. **MQTT**: + [MMQP](http://mqtt.org/) is a M2M "Internet of Things" connectivity protocol (ISO/IEC PRF 20922) designed to be extremely lightweight and ideal for mobile applications. Some broker implementations can be found [here](https://github.com/mqtt/mqtt.github.io/wiki/brokers). -## Gateway Specification + 3. **AMQP**: + [AMQP](https://www.amqp.org/) is a open standard messaging protocol (ISO/IEC 19464). There are a variety of broker implementations including, but not limited to the following: + - [Apache ActiveMQ](http://activemq.apache.org/) + - [Apache Qpid](https://qpid.apache.org/) + - [StormMQ](http://stormmq.com/) + - [RabbitMQ](https://www.rabbitmq.com/) + 4. **Kafka** + [Apache Kafka](https://kafka.apache.org/) is a distributed streaming platform. We use Shopify's [sarama](https://github.com/Shopify/sarama) client for consuming Kafka messages. -| Field | Description | -|----------------------|--------------| -| DeploySpec | Pod specification for gateway -| ConfigMap | Name of the configmap containing gateway configuration/s | -| Type | Type of gateway | -| EventVersion | To mark event version | -| ImageVersion | ImageVersion is the version for gateway components images to run | -| ServiceSpec | Specifications of the service to expose the gateway | -| Watchers | Watchers are components which are interested listening to notifications from the gateway | -| RPCPort | Used to communicate between gRPC gateway client and gRPC gateway server | -| HTTPServerPort | Used to communicate between gateway client and server over http | -| DispatchMechanism | Messaging mechanism used to send events from gateway to watchers | +### Community gateways +https://github.com/argoproj/argo-events/tree/master/gateways/community -## Gateway Deployment +In order to take advantage of the various gateway types, you may need to install compatible message platforms (e.g. amqp, mmqp, NATS, etc..) and s3 api compatible object storage servers (e.g. Minio, Rook, CEPH, NetApp). -All core gateways use kubernetes configmap to keep track of current gateway configurations. Multiple configurations can be defined for a single gateway and -each configuration will run in a separate go routine. The gateway watches updates to configmap which let us add new configuration at run time. +## Specification +https://github.com/argoproj/argo-events/blob/master/docs/gateway-protocol.md + +## Managing Event Sources +All gateways use kubernetes configmap to keep track of event sources. Multiple event source can be defined for a gateway. The gateway watches updates to configmap which let us add new event sources at run time. ## How to write a custom gateway? -Follow this tutorial to learn more -[Custom Gateways](custom-gateway.md) - - -## Gateway configurations -Gateway can have zero configuration(idle) or many configurations. A configuration can be added or removed during the runtime. - -### Calendars -Events produced can be based on a [cron](https://crontab.guru/) schedule or an [interval duration](https://golang.org/pkg/time/#ParseDuration). In addition, calendar gateway currently supports a `recurrence` field in which to specify special exclusion dates for which this gateway will not produce an event. -``` - calendar.fooConfig: |- - interval: 10s - calendar.barConfig: |- - schedule: 30 * * * * -``` - -### Webhooks -Webhook gateway expose a basic HTTP server endpoint/s. -Users can register multiple REST API endpoint. See Request Methods in RFC7231 to define the HTTP REST endpoint. - -``` - # portConfig defines port on which http server should run. Also it serves as index route. - webhook.portConfig: |- - port: "12000" - endpoint: "/" - method: "POST" - webhook.fooConfig: |- - endpoint: "/foo" - method: "POST" -``` - -### Kubernetes Resources -Resource gateway support watching Kubernetes resources. Users can specify `group`, `version`, `kind`, and filters including prefix of the object name, labels, annotations, and createdBy time. - -``` - resource.fooConfig: |- - namespace: argo-events - group: "argoproj.io" - version: "v1alpha1" - kind: "Workflow" - filter: - prefix: scripts-bash - labels: - workflows.argoproj.io/phase: Succeeded -``` - -### Artifacts -Artifact gateway support S3 `bucket-notifications` via [Minio](https://docs.minio.io/docs/minio-bucket-notification-guide). Note that a supported notification target must be running, exposed, and configured in the Minio server. For more information, please refer to the [artifact guide](artifact-guide.md). -``` - s3.fooConfig: |- - s3EventConfig: - bucket: foo - endpoint: minio.argo-events:9000 - event: s3:ObjectCreated:Put - filter: - prefix: "" - suffix: "" - insecure: true - accessKey: - key: accesskey - name: minio - secretKey: - key: secretkey - name: minio - s3.barConfig: |- - s3EventConfig: - bucket: bar - endpoint: minio.argo-events:9000 - event: s3:ObjectCreated:Get - filter: - prefix: "xyz" - suffix: "" - insecure: true - accessKey: - key: accesskey - name: minio - secretKey: - key: secretkey - name: minio -``` - -### Streams -Stream gateways contain a generic specification for messages received on a queue and/or though messaging server. The following are the `builtin` supported stream gateways. - -#### NATS -[Nats](https://nats.io/) is an open-sourced, lightweight, secure, and scalable messaging system for cloud native applications and microservices architecture. It is currently a hosted CNCF Project. -``` - nats.fooConfig: |- - url: nats://nats.argo-events:4222 - subject: foo - nats.barConfig: |- - url: nats://nats.argo-events:4222 - subject: bar -``` - - -#### MQTT -[MMQP](http://mqtt.org/) is a M2M "Internet of Things" connectivity protocol (ISO/IEC PRF 20922) designed to be extremely lightweight and ideal for mobile applications. Some broker implementations can be found [here](https://github.com/mqtt/mqtt.github.io/wiki/brokers). -``` - mqtt.fooConfig: |- - url: tcp://mqtt.argo-events:1883 - topic: foo - mqtt.barConfig: |- - url: tcp://mqtt.argo-events:1883 - topic: bar -``` - - -#### AMQP -[AMQP](https://www.amqp.org/) is a open standard messaging protocol (ISO/IEC 19464). There are a variety of broker implementations including, but not limited to the following: -- [Apache ActiveMQ](http://activemq.apache.org/) -- [Apache Qpid](https://qpid.apache.org/) -- [StormMQ](http://stormmq.com/) -- [RabbitMQ](https://www.rabbitmq.com/) -``` - amqp.fooConfig: |- - url: amqp://amqp.argo-events:5672 - exchangeName: fooExchangeName - exchangeType: fanout - routingKey: fooRoutingKey - amqp.barConfig: |- - url: amqp://amqp.argo-events:5672 - exchangeName: barExchangeName - exchangeType: fanout - routingKey: barRoutingKey -``` - - -#### Kafka -[Apache Kafka](https://kafka.apache.org/) is a distributed streaming platform. We use Shopify's [sarama](https://github.com/Shopify/sarama) client for consuming Kafka messages. -``` - kafka.fooConfig: |- - url: kafka.argo-events:9092 - attributes: - topic: foo - partition: "0" - kafka.barConfig: |- - url: kafka.argo-events:9092 - topic: bar - partition: "1" -``` - -### Examples -Explore [Gateway Examples](https://github.com/argoproj/argo-events/tree/master/examples/gateways) \ No newline at end of file +Follow [Custom Gateways](custom-gateway.md) + +## Examples +Explore [Gateway Examples](https://github.com/argoproj/argo-events/tree/master/examples/gateways) diff --git a/docs/gateway-protocol.md b/docs/gateway-protocol.md index bc9a89d875..8855d3d0d0 100644 --- a/docs/gateway-protocol.md +++ b/docs/gateway-protocol.md @@ -4,12 +4,15 @@ ## Table of Contents - [github.com/argoproj/argo-events/pkg/apis/gateway/v1alpha1/generated.proto](#github.com/argoproj/argo-events/pkg/apis/gateway/v1alpha1/generated.proto) + - [EventProtocol](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EventProtocol) - [Gateway](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Gateway) - [GatewayList](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.GatewayList) - [GatewayNotificationWatcher](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.GatewayNotificationWatcher) - [GatewaySpec](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.GatewaySpec) - [GatewayStatus](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.GatewayStatus) - [GatewayStatus.NodesEntry](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.GatewayStatus.NodesEntry) + - [Http](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Http) + - [Nats](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Nats) - [NodeStatus](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.NodeStatus) - [NotificationWatchers](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.NotificationWatchers) - [SensorNotificationWatcher](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.SensorNotificationWatcher) @@ -29,6 +32,23 @@ + + +### EventProtocol +Dispatch protocol contains configuration necessary to dispatch an event to sensor over different communication protocols + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| type | [string](#string) | optional | | +| http | [Http](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Http) | optional | | +| nats | [Nats](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Nats) | optional | | + + + + + + ### Gateway @@ -40,7 +60,7 @@ Gateway is the definition of a gateway resource | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| metadata | [k8s.io.apimachinery.pkg.apis.meta.v1.ObjectMeta](#k8s.io.apimachinery.pkg.apis.meta.v1.ObjectMeta) | optional | +k8s:openapi-gen=false | +| metadata | [k8s.io.apimachinery.pkg.apis.meta.v1.ObjectMeta](#k8s.io.apimachinery.pkg.apis.meta.v1.ObjectMeta) | optional | | | status | [GatewayStatus](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.GatewayStatus) | optional | | | spec | [GatewaySpec](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.GatewaySpec) | optional | | @@ -58,7 +78,7 @@ GatewayList is the list of Gateway resources | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| metadata | [k8s.io.apimachinery.pkg.apis.meta.v1.ListMeta](#k8s.io.apimachinery.pkg.apis.meta.v1.ListMeta) | optional | +k8s:openapi-gen=false | +| metadata | [k8s.io.apimachinery.pkg.apis.meta.v1.ListMeta](#k8s.io.apimachinery.pkg.apis.meta.v1.ListMeta) | optional | | | items | [Gateway](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Gateway) | repeated | | @@ -98,7 +118,7 @@ GatewaySpec represents gateway specifications | serviceSpec | [k8s.io.api.core.v1.Service](#k8s.io.api.core.v1.Service) | optional | ServiceSpec is the specifications of the service to expose the gateway Refer https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#service-v1-core | | watchers | [NotificationWatchers](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.NotificationWatchers) | optional | Watchers are components which are interested listening to notifications from this gateway These only need to be specified when gateway dispatch mechanism is through HTTP POST notifications. In future, support for NATS, KAFKA will be added as a means to dispatch notifications in which case specifying watchers would be unnecessary. | | processorPort | [string](#string) | optional | Port on which the gateway event source processor is running on. | -| dispatchProtocol | [string](#string) | optional | DispatchProtocol is the underlying protocol used to send events from gateway to watchers(components interested in listening to event from this gateway) | +| eventProtocol | [EventProtocol](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EventProtocol) | optional | EventProtocol is the underlying protocol used to send events from gateway to watchers(components interested in listening to event from this gateway) | @@ -139,6 +159,39 @@ GatewayStatus contains information about the status of a gateway. + + +### Http + + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| port | [string](#string) | optional | | + + + + + + + + +### Nats + + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| url | [string](#string) | optional | | +| type | [string](#string) | optional | | +| clientId | [string](#string) | optional | | +| clusterId | [string](#string) | optional | | + + + + + + ### NodeStatus @@ -148,13 +201,13 @@ A single node can represent one configuration. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| id | [string](#string) | optional | ID is a unique identifier of a node within a gateway It is a hash of the node name | +| id | [string](#string) | optional | ID is a unique identifier of a node within a sensor It is a hash of the node name | | name | [string](#string) | optional | Name is a unique name in the node tree used to generate the node ID | | displayName | [string](#string) | optional | DisplayName is the human readable representation of the node | | phase | [string](#string) | optional | Phase of the node | | startedAt | [k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime](#k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime) | optional | StartedAt is the time at which this node started +k8s:openapi-gen=false | | message | [string](#string) | optional | Message store data or something to save for configuration | -| updateTime | [k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime](#k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime) | optional | UpdateTime is the time when node(gateway configuration) was updated +k8s:openapi-gen=false | +| updateTime | [k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime](#k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime) | optional | UpdateTime is the time when node(gateway configuration) was updated | @@ -180,12 +233,12 @@ NotificationWatchers are components which are interested listening to notificati ### SensorNotificationWatcher -SensorNotificationWatcher is the gateway interested in listening to notifications from this gateway +SensorNotificationWatcher is the sensor interested in listening to notifications from this gateway | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| name | [string](#string) | optional | Name is name of the gateway | +| name | [string](#string) | optional | Name is name of the sensor | diff --git a/docs/grpc-gateway.png b/docs/grpc-gateway.png deleted file mode 100644 index 3091f6f99f..0000000000 Binary files a/docs/grpc-gateway.png and /dev/null differ diff --git a/docs/http-gateway.png b/docs/http-gateway.png deleted file mode 100644 index 56f53c772c..0000000000 Binary files a/docs/http-gateway.png and /dev/null differ diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 2255f58c34..0000000000 --- a/docs/index.md +++ /dev/null @@ -1,28 +0,0 @@ -# Argo Events Documentation - -## Why Argo Events? -- Containers. Designed from the ground-up as Kubernetes-native. -- Extremely lightweight. All gateways, with exception of calendar based gateway, are event-driven, meaning there is no polling involved. -- Configurable. Configure gateways at runtime -- Extensible. Write custom gateways for any business use case in the language of your choice -- Scalability & Resilient. -- Simple or Complex dependencies. Manage everything from simple, linear, real-time dependencies to complex, multi-source batch job dependencies. - -## Basics -Argo Events is an open source event-based dependency manager for Kubernetes. The core concept of the project are - * `gateways` which are implemented as a Kubernetes-native Custom Resource Definition that either produce the events internally or process the events that originate from outside the gateway - - * `sensors` which are implemented as a Kubernetes-native Custom Resource Definition that define a set of dependencies - (inputs) and actions (outputs). The sensor's actions will only be triggered after it's dependencies have been resolved. - - Define gateway to support business-level logic for producing events. - - Define multiple dependencies from a variety of sources - - Trigger messages and Kubernetes object creation after successful dependency resolution - - Trigger escalation after errors, or dependency constraint failures - - Build and manage a distributed, cross-team, event-driven architecture - - Easily leverage Kubernetes native APIs to monitor dependencies - -## Learn More -- [Quickstart](quickstart.md) -- [Gateways](gateway-guide.md) -- [Sensors](sensor-guide.md) -- [Triggers](trigger-guide.md) \ No newline at end of file diff --git a/docs/quickstart.md b/docs/quickstart.md index 36551808cd..fe7484fae9 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -13,7 +13,7 @@ go get github.com/argoproj/argo-events cd $GOPATH/src/github.com/argoproj/argo-events ``` -## 2. Deploy Argo Events SA, ClusterRoles, ConfigMap, Sensor Controller and Gateway Controller +## 2. Deploy Argo Events SA, Roles, ConfigMap, Sensor Controller and Gateway Controller Note: Modify the [argo-events-cluster-roles.yaml](../hack/k8s/manifests/argo-events-cluster-roles.yaml) file to use the correct namespace that you wish to deploy the sensor controller and gateway controller. ``` @@ -39,12 +39,12 @@ Follow instructions from https://github.com/argoproj/argo/blob/master/demo.md ## 4. Create a webhook gateway ``` kubectl apply -n argo-events -f examples/gateways/webhook-gateway-configmap.yaml -kubectl apply -n argo-events -f examples/gateways/webhook.yaml +kubectl apply -n argo-events -f examples/gateways/webhook-http.yaml ``` ## 5. Create a webhook sensor ``` -kubectl apply -n argo-events -f examples/sensors/webhook.yaml +kubectl apply -n argo-events -f examples/sensors/webhook-http.yaml ``` ## 6. Trigger the webhook & corresponding Argo workflow @@ -52,15 +52,15 @@ Trigger the webhook via sending a http POST request to `/foo` endpoint. You can gateway configuration at run time as well. Note: the `WEBHOOK_SERVICE_URL` will differ based on the Kubernetes cluster. ``` -export WEBHOOK_SERVICE_URL=$(minikube service -n argo-events --url webhook-gateway-gateway-svc) +export WEBHOOK_SERVICE_URL=$(minikube service -n argo-events --url ) echo $WEBHOOK_SERVICE_URL curl -d '{"message":"this is my first webhook"}' -H "Content-Type: application/json" -X POST $WEBHOOK_SERVICE_URL/foo ``` Note: - * If you are facing an issue getting service url by running `minikube service -n argo-events --url webhook-gateway-gateway-svc`, you can use `kubectl port-forward` + * If you are facing an issue getting service url by running `minikube service -n argo-events --url `, you can use `kubectl port-forward` * Open another terminal window and enter `kubectl port-forward -n argo-events 9003:` - * You can now user `localhost:9003` to query webhook gateway + * You can now use `localhost:9003` to query webhook gateway Verify that the Argo workflow was run when the trigger was executed. @@ -68,13 +68,4 @@ Verify that the Argo workflow was run when the trigger was executed. argo list -n argo-events ``` -Verify that the sensor was updated correctly and moved to a "Complete" phase. -``` -kubectl get sensor -n argo-events webhook-sensor -o yaml -``` - -Check the logs of the sensor-controller pod, gateway-controller, associated gateways and sensors if there are problems. - -## 7. Next steps -* [Follow tutorial on gateways and sensors](tutorial.md) -* Write your first gateway. Follow the tutorial [Custom Gateways](custom-gateway.md). \ No newline at end of file +To debug, check the logs of the sensor-controller pod, gateway-controller, associated gateways and sensors. diff --git a/docs/sensor-fsm.png b/docs/sensor-fsm.png deleted file mode 100644 index f69aaa4c21..0000000000 Binary files a/docs/sensor-fsm.png and /dev/null differ diff --git a/docs/sensor-guide.md b/docs/sensor-guide.md index cb6837dd4f..461c9428a5 100644 --- a/docs/sensor-guide.md +++ b/docs/sensor-guide.md @@ -1,54 +1,32 @@ -## Sensor Guide - +# Sensor Guide Sensors define a set of dependencies (inputs) and actions (outputs). The sensor's actions will only be triggered after it's dependencies have been resolved. [Sensor examples](https://github.com/argoproj/argo-events/tree/master/examples/sensors) +1. [Specifications](#specifications) +2. [Dependencies](#dependencies) +3. [Triggers](#triggers) +4. [Passing event payload to trigger/s](#passing-event-payload-to-triggers) +5. [Filters](#filters) -### Dependencies -Dependencies(also called signals) are defined as "gateway-name/specific-configuration" -``` -signals: - - name: calendar-gateway/calendar.fooConfig - - name: webhook-gateway/webhook.barConfig -``` +## Specifications +https://github.com/argoproj/argo-events/blob/master/docs/sensor-protocol.md -### Repeating the sensor -Sensor can be configured to rerun by setting repeat property to `true` -``` -spec: - repeat: true -``` +## Dependencies +A sensor can contain list of event dependencies and a dependency is defined as "gateway-name:event-source-name". -### Triggers -Refer [Triggers](trigger-guide.md) guide. +**Note**: If a sensor defines more than one event dependencies then it will wait for all of these events to happen, +then only it will trigger workflow/s. Basically, waiting for event dependencies is a `AND` operation. +## Triggers +Refer [Triggers](trigger-guide.md) guide. -### Event payload -Event payload of any signal can be passed to any trigger. To pass complete payload without applying any filter, -do not set ```path``` -e.g. -```yaml -parameters: - - src: - signal: webhook-gateway/webhook.fooConfig - path: - dest: spec.templates.0.container.args.0 -``` - -Complete example to pass payload from signal to trigger can be found [here](https://github.com/argoproj/argo-events/blob/master/examples/sensors/webhook.yaml) - -To pass a particular field from payload to trigger, set ```path```. e.g. -```yaml -parameters: - - src: - signal: webhook-gateway/webhook.fooConfig - path: name - dest: spec.templates.0.container.args.0 -``` +## Passing event payload to trigger/s +Event payload of any signal can be passed to any trigger. -In above example, the object/value corresponding to key ```name``` will be passed to trigger. +* Pass complete event payload https://github.com/argoproj/argo-events/blob/master/examples/sensors/webhook-with-complete-payload.yaml +* Extract particular key from event payload and pass to trigger https://github.com/argoproj/argo-events/blob/master/examples/sensors/webhook-with-resource-param.yaml -### Filters +## Filters Additionally, you can apply filters on the payload. There are 3 types of filters: @@ -59,7 +37,7 @@ There are 3 types of filters: | EventContext | Filters metadata that provides circumstantial information about the signal. | | Data | Describes constraints and filters for payload | -#### Time Filter +### Time Filter ``` filters: time: @@ -67,7 +45,7 @@ filters: stop: "2020-01-02T15:04:05Z07:00" ``` -#### EventContext Filter +### EventContext Filter ``` filters: context: @@ -76,7 +54,7 @@ filters: contentType: application/json ``` -#### Data filter +### Data filter ``` filters: data: diff --git a/docs/sensor-pod-fsm.png b/docs/sensor-pod-fsm.png deleted file mode 100644 index 9f3d602afc..0000000000 Binary files a/docs/sensor-pod-fsm.png and /dev/null differ diff --git a/docs/sensor-protocol.md b/docs/sensor-protocol.md index afef8461b2..f0fd74dbb2 100644 --- a/docs/sensor-protocol.md +++ b/docs/sensor-protocol.md @@ -3,37 +3,32 @@ ## Table of Contents -- [github.com/argoproj/argo-events/pkg/apis/gateway/v1alpha1/generated.proto](#github.com/argoproj/argo-events/pkg/apis/gateway/v1alpha1/generated.proto) - - [ArtifactLocation](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.ArtifactLocation) - - [ConfigmapArtifact](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.ConfigmapArtifact) - - [Data](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Data) - - [DataFilter](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.DataFilter) - - [EscalationPolicy](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EscalationPolicy) - - [Event](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Event) - - [EventContext](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EventContext) - - [EventContext.ExtensionsEntry](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EventContext.ExtensionsEntry) - - [EventDependency](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EventDependency) - - [FileArtifact](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.FileArtifact) - - [GroupVersionKind](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.GroupVersionKind) - - [NodeStatus](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.NodeStatus) - - [ResourceObject](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.ResourceObject) - - [ResourceObject.LabelsEntry](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.ResourceObject.LabelsEntry) - - [ResourceParameter](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.ResourceParameter) - - [ResourceParameterSource](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.ResourceParameterSource) - - [RetryStrategy](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.RetryStrategy) - - [S3Artifact](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.S3Artifact) - - [S3Bucket](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.S3Bucket) - - [S3Filter](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.S3Filter) - - [Sensor](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Sensor) - - [SensorList](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.SensorList) - - [SensorSpec](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.SensorSpec) - - [SensorStatus](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.SensorStatus) - - [SensorStatus.NodesEntry](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.SensorStatus.NodesEntry) - - [SignalFilter](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.SignalFilter) - - [TimeFilter](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.TimeFilter) - - [Trigger](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Trigger) - - [URI](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.URI) - - [URLArtifact](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.URLArtifact) +- [github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1/generated.proto](#github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1/generated.proto) + - [ArtifactLocation](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.ArtifactLocation) + - [ConfigmapArtifact](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.ConfigmapArtifact) + - [Data](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Data) + - [DataFilter](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.DataFilter) + - [EventDependency](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.EventDependency) + - [EventDependencyFilter](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.EventDependencyFilter) + - [EventProtocol](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.EventProtocol) + - [FileArtifact](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.FileArtifact) + - [GroupVersionKind](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.GroupVersionKind) + - [Http](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Http) + - [Nats](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Nats) + - [NodeStatus](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.NodeStatus) + - [ResourceObject](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.ResourceObject) + - [ResourceObject.LabelsEntry](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.ResourceObject.LabelsEntry) + - [ResourceParameter](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.ResourceParameter) + - [ResourceParameterSource](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.ResourceParameterSource) + - [RetryStrategy](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.RetryStrategy) + - [Sensor](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Sensor) + - [SensorList](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.SensorList) + - [SensorSpec](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.SensorSpec) + - [SensorStatus](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.SensorStatus) + - [SensorStatus.NodesEntry](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.SensorStatus.NodesEntry) + - [TimeFilter](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.TimeFilter) + - [Trigger](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Trigger) + - [URLArtifact](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.URLArtifact) @@ -43,14 +38,14 @@ - +

Top

-## github.com/argoproj/argo-events/pkg/apis/gateway/v1alpha1/generated.proto +## github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1/generated.proto - + ### ArtifactLocation ArtifactLocation describes the source location for an external artifact @@ -58,18 +53,18 @@ ArtifactLocation describes the source location for an external artifact | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| s3 | [S3Artifact](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.S3Artifact) | optional | | +| s3 | [github.com.argoproj.argo_events.pkg.apis.common.S3Artifact](#github.com.argoproj.argo_events.pkg.apis.common.S3Artifact) | optional | | | inline | [string](#string) | optional | | -| file | [FileArtifact](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.FileArtifact) | optional | | -| url | [URLArtifact](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.URLArtifact) | optional | | -| configmap | [ConfigmapArtifact](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.ConfigmapArtifact) | optional | | +| file | [FileArtifact](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.FileArtifact) | optional | | +| url | [URLArtifact](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.URLArtifact) | optional | | +| configmap | [ConfigmapArtifact](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.ConfigmapArtifact) | optional | | - + ### ConfigmapArtifact ConfigmapArtifact contains information about artifact in k8 configmap @@ -86,7 +81,7 @@ ConfigmapArtifact contains information about artifact in k8 configmap - + ### Data @@ -94,15 +89,14 @@ ConfigmapArtifact contains information about artifact in k8 configmap | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| filters | [DataFilter](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.DataFilter) | repeated | filter constraints | -| escalationPolicy | [EscalationPolicy](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EscalationPolicy) | optional | EscalationPolicy is the escalation to trigger in case the eventDependency filter fails | +| filters | [DataFilter](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.DataFilter) | repeated | filter constraints | - + ### DataFilter DataFilter describes constraints and filters for event data @@ -115,164 +109,162 @@ See Rob Pike's Post: https://commandcenter.blogspot.com/2011/08/regular-expr | path | [string](#string) | optional | Path is the JSONPath of the event's (JSON decoded) data key Path is a series of keys separated by a dot. A key may contain wildcard characters '*' and '?'. To access an array value use the index as the key. The dot and wildcard characters can be escaped with '\\'. See https://github.com/tidwall/gjson#path-syntax for more information on how to use this. | | type | [string](#string) | optional | Type contains the JSON type of the data | | value | [string](#string) | optional | Value is the expected string value for this key Booleans are pased using strconv.ParseBool() Numbers are parsed using as float64 using strconv.ParseFloat() Strings are taken as is Nils this value is ignored | -| escalationPolicy | [EscalationPolicy](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EscalationPolicy) | optional | EscalationPolicy is the escalation to trigger in case the eventDependency filter fails | - + -### EscalationPolicy -EscalationPolicy describes the policy for escalating sensors in an Error state. -An escalation policy is associated with eventDependency filter. Whenever a eventDependency filter fails -escalation will be triggered +### EventDependency +EventDependency describes a dependency | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| name | [string](#string) | optional | Name is name of the escalation policy This is referred by eventDependency filter/s | -| level | [string](#string) | optional | Level is the degree of importance | -| message | [string](#string) | optional | need someway to progressively get more serious notifications | +| name | [string](#string) | optional | Name is a unique name of this dependency | +| deadline | [int64](#int64) | optional | Deadline is the duration in seconds after the StartedAt time of the sensor after which this event is terminated. Note: this functionality is not yet respected, but it's theoretical behavior is as follows: This trumps the recurrence patterns of calendar events and allows any event to have a strict defined life. After the deadline is reached and this event has not in a Resolved state, this event is marked as Failed and proper escalations should proceed. | +| filters | [EventDependencyFilter](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.EventDependencyFilter) | optional | Filters and rules governing tolerations of success and constraints on the context and data of an event | +| connected | [bool](#bool) | optional | Connected tells if subscription is already setup in case of nats protocol. | - + -### Event -Event is a data record expressing an occurrence and its context. -Adheres to the CloudEvents v0.1 specification +### EventDependencyFilter +EventDependencyFilter defines filters and constraints for a event. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| context | [EventContext](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EventContext) | optional | | -| data | [bytes](#bytes) | optional | | +| name | [string](#string) | optional | Name is the name of event filter | +| time | [TimeFilter](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.TimeFilter) | optional | Time filter on the event with escalation | +| context | [github.com.argoproj.argo_events.pkg.apis.common.EventContext](#github.com.argoproj.argo_events.pkg.apis.common.EventContext) | optional | Context filter constraints with escalation | +| data | [Data](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Data) | optional | Data filter constraints with escalation | - + -### EventContext -EventContext contains metadata that provides circumstantial information about the occurrence. +### EventProtocol +EventProtocol contains configuration necessary to receieve an event from gateway over different communication protocols | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| eventType | [string](#string) | optional | The type of occurrence which has happened. Often this attribute is used for routing, observability, policy enforcement, etc. should be prefixed with a reverse-DNS name. The prefixed domain dictates the organization which defines the semantics of this event type. ex: com.github.pull.create | -| eventTypeVersion | [string](#string) | optional | The version of the eventType. Enables the interpretation of data by eventual consumers, requires the consumer to be knowledgeable about the producer. | -| cloudEventsVersion | [string](#string) | optional | The version of the CloudEvents specification which the event uses. Enables the interpretation of the context. | -| source | [URI](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.URI) | optional | This describes the event producer. | -| eventID | [string](#string) | optional | ID of the event. The semantics are explicitly undefined to ease the implementation of producers. Enables deduplication. Must be unique within scope of producer. | -| eventTime | [k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime](#k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime) | optional | Timestamp of when the event happened. Must adhere to format specified in RFC 3339. +k8s:openapi-gen=false | -| schemaURL | [URI](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.URI) | optional | A link to the schema that the data attribute adheres to. Must adhere to the format specified in RFC 3986. | -| contentType | [string](#string) | optional | Content type of the data attribute value. Enables the data attribute to carry any type of content, whereby format and encoding might differ from that of the chosen event format. For example, the data attribute may carry an XML or JSON payload and the consumer is informed by this attribute being set to "application/xml" or "application/json" respectively. | -| extensions | [EventContext.ExtensionsEntry](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EventContext.ExtensionsEntry) | repeated | This is for additional metadata and does not have a mandated structure. Enables a place for custom fields a producer or middleware might want to include and provides a place to test metadata before adding them to the CloudEvents specification. | -| escalationPolicy | [EscalationPolicy](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EscalationPolicy) | optional | EscalationPolicy is the name of escalation policy to trigger in case the eventDependency filter fails | +| type | [string](#string) | optional | Type defines the type of protocol over which events will be receieved | +| http | [Http](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Http) | optional | Http contains the information required to setup a http server and listen to incoming events | +| nats | [Nats](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Nats) | optional | Nats contains the information required to connect to nats server and get subscriptions | - - -### EventContext.ExtensionsEntry + +### FileArtifact +FileArtifact contains information about an artifact in a filesystem | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| key | [string](#string) | optional | | -| value | [string](#string) | optional | | +| path | [string](#string) | optional | | - + -### EventDependency -EventDependency describes a dependency +### GroupVersionKind +GroupVersionKind unambiguously identifies a kind. It doesn't anonymously include GroupVersion +to avoid automatic coercion. It doesn't use a GroupVersion to avoid custom marshalling. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| name | [string](#string) | optional | Name is a unique name of this dependency | -| deadline | [int64](#int64) | optional | Deadline is the duration in seconds after the StartedAt time of the gateway after which this eventDependency is terminated. Note: this functionality is not yet respected, but it's theoretical behavior is as follows: This trumps the recurrence patterns of calendar signals and allows any eventDependency to have a strict defined life. After the deadline is reached and this eventDependency has not in a Resolved state, this eventDependency is marked as Failed and proper escalations should proceed. | -| filters | [SignalFilter](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.SignalFilter) | optional | Filters and rules governing tolerations of success and constraints on the context and data of an event | +| group | [string](#string) | optional | | +| version | [string](#string) | optional | | +| kind | [string](#string) | optional | | - + -### FileArtifact -FileArtifact contains information about an artifact in a filesystem +### Http +Http contains the information required to setup a http server and listen to incoming events | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| path | [string](#string) | optional | | +| port | [string](#string) | optional | Port on which server will run | - + -### GroupVersionKind -GroupVersionKind unambiguously identifies a kind. It doesn't anonymously include GroupVersion -to avoid automatic coercion. It doesn't use a GroupVersion to avoid custom marshalling. +### Nats +Nats contains the information required to connect to nats server and get subscriptions | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| group | [string](#string) | optional | | -| version | [string](#string) | optional | | -| kind | [string](#string) | optional | | +| url | [string](#string) | optional | URL is nats server/service URL | +| startWithLastReceived | [bool](#bool) | optional | Subscribe starting with most recently published value. Refer https://github.com/nats-io/go-nats-streaming | +| deliverAllAvailable | [bool](#bool) | optional | Receive all stored values in order. | +| startAtSequence | [string](#string) | optional | Receive messages starting at a specific sequence number | +| startAtTime | [string](#string) | optional | Subscribe starting at a specific time | +| startAtTimeDelta | [string](#string) | optional | Subscribe starting a specific amount of time in the past (e.g. 30 seconds ago) | +| durable | [bool](#bool) | optional | Durable subscriptions allow clients to assign a durable name to a subscription when it is created | +| clusterId | [string](#string) | optional | The NATS Streaming cluster ID | +| clientId | [string](#string) | optional | The NATS Streaming cluster ID | +| type | [string](#string) | optional | Type of the connection. either standard or streaming | - + ### NodeStatus -NodeStatus describes the status for an individual node in the gateway's FSM. -A single node can represent the status for eventDependency or a trigger. +NodeStatus describes the status for an individual node in the sensor's FSM. +A single node can represent the status for event or a trigger. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| id | [string](#string) | optional | ID is a unique identifier of a node within a gateway It is a hash of the node name | +| id | [string](#string) | optional | ID is a unique identifier of a node within a sensor It is a hash of the node name | | name | [string](#string) | optional | Name is a unique name in the node tree used to generate the node ID | | displayName | [string](#string) | optional | DisplayName is the human readable representation of the node | | type | [string](#string) | optional | Type is the type of the node | | phase | [string](#string) | optional | Phase of the node | -| startedAt | [k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime](#k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime) | optional | StartedAt is the time at which this node started +k8s:openapi-gen=false | -| completedAt | [k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime](#k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime) | optional | CompletedAt is the time at which this node completed +k8s:openapi-gen=false | -| message | [string](#string) | optional | store data or something to save for eventDependency notifications or trigger events | -| latestEvent | [Event](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Event) | optional | Event stores the last seen event for this node | +| startedAt | [k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime](#k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime) | optional | StartedAt is the time at which this node started | +| completedAt | [k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime](#k8s.io.apimachinery.pkg.apis.meta.v1.MicroTime) | optional | CompletedAt is the time at which this node completed | +| message | [string](#string) | optional | store data or something to save for event notifications or trigger events | +| event | [github.com.argoproj.argo_events.pkg.apis.common.Event](#github.com.argoproj.argo_events.pkg.apis.common.Event) | optional | Event stores the last seen event for this node | - + ### ResourceObject ResourceObject is the resource object to create on kubernetes @@ -280,18 +272,18 @@ ResourceObject is the resource object to create on kubernetes | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| groupVersionKind | [GroupVersionKind](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.GroupVersionKind) | optional | The unambiguous kind of this object - used in order to retrieve the appropriate kubernetes api client for this resource | -| namespace | [string](#string) | optional | Namespace in which to create this object optional defaults to the service account namespace | -| source | [ArtifactLocation](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.ArtifactLocation) | optional | Source of the K8 resource file(s) | -| labels | [ResourceObject.LabelsEntry](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.ResourceObject.LabelsEntry) | repeated | Map of string keys and values that can be used to organize and categorize (scope and select) objects. This overrides any labels in the unstructured object with the same key. | -| parameters | [ResourceParameter](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.ResourceParameter) | repeated | Parameters is the list of resource parameters to pass in the object | +| groupVersionKind | [GroupVersionKind](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.GroupVersionKind) | optional | The unambiguous kind of this object - used in order to retrieve the appropriate kubernetes api client for this resource | +| namespace | [string](#string) | optional | Namespace in which to create this object defaults to the service account namespace | +| source | [ArtifactLocation](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.ArtifactLocation) | optional | Source of the K8 resource file(s) | +| labels | [ResourceObject.LabelsEntry](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.ResourceObject.LabelsEntry) | repeated | Map of string keys and values that can be used to organize and categorize (scope and select) objects. This overrides any labels in the unstructured object with the same key. | +| parameters | [ResourceParameter](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.ResourceParameter) | repeated | Parameters is the list of resource parameters to pass in the object | - + ### ResourceObject.LabelsEntry @@ -307,7 +299,7 @@ ResourceObject is the resource object to create on kubernetes - + ### ResourceParameter ResourceParameter indicates a passed parameter to a service template @@ -315,7 +307,7 @@ ResourceParameter indicates a passed parameter to a service template | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| src | [ResourceParameterSource](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.ResourceParameterSource) | optional | Src contains a source reference to the value of the resource parameter from a eventDependency event | +| src | [ResourceParameterSource](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.ResourceParameterSource) | optional | Src contains a source reference to the value of the resource parameter from a event event | | dest | [string](#string) | optional | Dest is the JSONPath of a resource key. A path is a series of keys separated by a dot. The colon character can be escaped with '.' The -1 key can be used to append a value to an existing array. See https://github.com/tidwall/sjson#path-syntax for more information about how this is used. | @@ -323,15 +315,15 @@ ResourceParameter indicates a passed parameter to a service template - + ### ResourceParameterSource -ResourceParameterSource defines the source for a resource parameter from a eventDependency event +ResourceParameterSource defines the source for a resource parameter from a event event | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| eventDependency | [string](#string) | optional | EventDependency is the name of the eventDependency for which to retrieve this event | +| event | [string](#string) | optional | Event is the name of the event for which to retrieve this event | | path | [string](#string) | optional | Path is the JSONPath of the event's (JSON decoded) data key Path is a series of keys separated by a dot. A key may contain wildcard characters '*' and '?'. To access an array value use the index as the key. The dot and wildcard characters can be escaped with '\\'. See https://github.com/tidwall/gjson#path-syntax for more information on how to use this. | | value | [string](#string) | optional | Value is the default literal value to use for this parameter source This is only used if the path is invalid. If the path is invalid and this is not defined, this param source will produce an error. | @@ -340,7 +332,7 @@ ResourceParameterSource defines the source for a resource parameter from a event - + ### RetryStrategy RetryStrategy represents a strategy for retrying operations @@ -351,64 +343,10 @@ TODO: implement me - - -### S3Artifact -S3Artifact contains information about an artifact in S3 - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| s3Bucket | [S3Bucket](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.S3Bucket) | optional | | -| key | [string](#string) | optional | | -| event | [string](#string) | optional | | -| filter | [S3Filter](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.S3Filter) | optional | | - - - - - - - - -### S3Bucket -S3Bucket contains information for an S3 Bucket - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| endpoint | [string](#string) | optional | | -| bucket | [string](#string) | optional | | -| region | [string](#string) | optional | | -| insecure | [bool](#bool) | optional | | -| accessKey | [k8s.io.api.core.v1.SecretKeySelector](#k8s.io.api.core.v1.SecretKeySelector) | optional | +k8s:openapi-gen=false | -| secretKey | [k8s.io.api.core.v1.SecretKeySelector](#k8s.io.api.core.v1.SecretKeySelector) | optional | +k8s:openapi-gen=false | - - - - - - - - -### S3Filter -S3Filter represents filters to apply to bucket nofifications for specifying constraints on objects - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| prefix | [string](#string) | optional | | -| suffix | [string](#string) | optional | | - - - - - - - + ### Sensor -Sensor is the definition of a gateway resource +Sensor is the definition of a sensor resource +genclient +genclient:noStatus +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -417,16 +355,16 @@ Sensor is the definition of a gateway resource | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| metadata | [k8s.io.apimachinery.pkg.apis.meta.v1.ObjectMeta](#k8s.io.apimachinery.pkg.apis.meta.v1.ObjectMeta) | optional | +k8s:openapi-gen=false | -| spec | [SensorSpec](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.SensorSpec) | optional | | -| status | [SensorStatus](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.SensorStatus) | optional | | +| metadata | [k8s.io.apimachinery.pkg.apis.meta.v1.ObjectMeta](#k8s.io.apimachinery.pkg.apis.meta.v1.ObjectMeta) | optional | | +| spec | [SensorSpec](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.SensorSpec) | optional | | +| status | [SensorStatus](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.SensorStatus) | optional | | - + ### SensorList SensorList is the list of Sensor resources @@ -435,52 +373,53 @@ SensorList is the list of Sensor resources | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| metadata | [k8s.io.apimachinery.pkg.apis.meta.v1.ListMeta](#k8s.io.apimachinery.pkg.apis.meta.v1.ListMeta) | optional | +k8s:openapi-gen=false | -| items | [Sensor](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Sensor) | repeated | | +| metadata | [k8s.io.apimachinery.pkg.apis.meta.v1.ListMeta](#k8s.io.apimachinery.pkg.apis.meta.v1.ListMeta) | optional | | +| items | [Sensor](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Sensor) | repeated | | - + ### SensorSpec -SensorSpec represents desired gateway state +SensorSpec represents desired sensor state | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| dependencies | [EventDependency](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EventDependency) | repeated | EventDependency is a list of the events that this gateway is dependent on. | -| triggers | [Trigger](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Trigger) | repeated | Triggers is a list of the things that this gateway evokes. These are the outputs from this gateway. | -| deploySpec | [k8s.io.api.core.v1.PodSpec](#k8s.io.api.core.v1.PodSpec) | optional | DeploySpec contains gateway pod specification. For more information, read https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#pod-v1-core | +| dependencies | [EventDependency](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.EventDependency) | repeated | Dependencies is a list of the events that this sensor is dependent on. | +| triggers | [Trigger](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Trigger) | repeated | Triggers is a list of the things that this sensor evokes. These are the outputs from this sensor. | +| deploySpec | [k8s.io.api.core.v1.PodSpec](#k8s.io.api.core.v1.PodSpec) | optional | DeploySpec contains sensor pod specification. For more information, read https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#pod-v1-core | +| eventProtocol | [EventProtocol](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.EventProtocol) | optional | EventProtocol is the protocol through which sensor receives events from gateway | - + ### SensorStatus -SensorStatus contains information about the status of a gateway. +SensorStatus contains information about the status of a sensor. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| phase | [string](#string) | optional | Phase is the high-level summary of the gateway | -| startedAt | [k8s.io.apimachinery.pkg.apis.meta.v1.Time](#k8s.io.apimachinery.pkg.apis.meta.v1.Time) | optional | StartedAt is the time at which this gateway was initiated +k8s:openapi-gen=false | -| completedAt | [k8s.io.apimachinery.pkg.apis.meta.v1.Time](#k8s.io.apimachinery.pkg.apis.meta.v1.Time) | optional | CompletedAt is the time at which this gateway was completed +k8s:openapi-gen=false | -| completionCount | [int32](#int32) | optional | CompletionCount is the count of gateway's successful runs. | -| message | [string](#string) | optional | Message is a human readable string indicating details about a gateway in its phase | -| nodes | [SensorStatus.NodesEntry](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.SensorStatus.NodesEntry) | repeated | Nodes is a mapping between a node ID and the node's status it records the states for the FSM of this gateway. | +| phase | [string](#string) | optional | Phase is the high-level summary of the sensor | +| startedAt | [k8s.io.apimachinery.pkg.apis.meta.v1.Time](#k8s.io.apimachinery.pkg.apis.meta.v1.Time) | optional | StartedAt is the time at which this sensor was initiated | +| completedAt | [k8s.io.apimachinery.pkg.apis.meta.v1.Time](#k8s.io.apimachinery.pkg.apis.meta.v1.Time) | optional | CompletedAt is the time at which this sensor was completed | +| completionCount | [int32](#int32) | optional | CompletionCount is the count of sensor's successful runs. | +| message | [string](#string) | optional | Message is a human readable string indicating details about a sensor in its phase | +| nodes | [SensorStatus.NodesEntry](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.SensorStatus.NodesEntry) | repeated | Nodes is a mapping between a node ID and the node's status it records the states for the FSM of this sensor. | - + ### SensorStatus.NodesEntry @@ -489,52 +428,33 @@ SensorStatus contains information about the status of a gateway. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | | key | [string](#string) | optional | | -| value | [NodeStatus](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.NodeStatus) | optional | | - - +| value | [NodeStatus](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.NodeStatus) | optional | | - -### SignalFilter -SignalFilter defines filters and constraints for a eventDependency. - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| name | [string](#string) | optional | Name is the name of eventDependency filter | -| time | [TimeFilter](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.TimeFilter) | optional | Time filter on the eventDependency with escalation | -| context | [EventContext](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EventContext) | optional | Context filter constraints with escalation | -| data | [Data](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.Data) | optional | Data filter constraints with escalation | - - - - - - - + ### TimeFilter TimeFilter describes a window in time. -Filters out eventDependency events that occur outside the time limits. +Filters out event events that occur outside the time limits. In other words, only events that occur after Start and before Stop will pass this filter. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| start | [string](#string) | optional | Start is the beginning of a time window. Before this time, events for this eventDependency are ignored and format is hh:mm:ss | -| stop | [string](#string) | optional | StopPattern is the end of a time window. After this time, events for this eventDependency are ignored and format is hh:mm:ss | -| escalationPolicy | [EscalationPolicy](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.EscalationPolicy) | optional | EscalationPolicy is the escalation to trigger in case the eventDependency filter fails | +| start | [string](#string) | optional | Start is the beginning of a time window. Before this time, events for this event are ignored and format is hh:mm:ss | +| stop | [string](#string) | optional | StopPattern is the end of a time window. After this time, events for this event are ignored and format is hh:mm:ss | - + ### Trigger Trigger is an action taken, output produced, an event created, a message sent @@ -543,38 +463,16 @@ Trigger is an action taken, output produced, an event created, a message sent | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | | name | [string](#string) | optional | Name is a unique name of the action to take | -| resource | [ResourceObject](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.ResourceObject) | optional | Resource describes the resource that will be created by this action | +| resource | [ResourceObject](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.ResourceObject) | optional | Resource describes the resource that will be created by this action | | message | [string](#string) | optional | Message describes a message that will be sent on a queue | -| replyStrategy | [RetryStrategy](#github.com.argoproj.argo_events.pkg.apis.gateway.v1alpha1.RetryStrategy) | optional | RetryStrategy is the strategy to retry a trigger if it fails | - - - - - - - - -### URI -URI is a Uniform Resource Identifier based on RFC 3986 - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| scheme | [string](#string) | optional | | -| user | [string](#string) | optional | | -| password | [string](#string) | optional | | -| host | [string](#string) | optional | | -| port | [int32](#int32) | optional | | -| path | [string](#string) | optional | | -| query | [string](#string) | optional | | -| fragment | [string](#string) | optional | | +| replyStrategy | [RetryStrategy](#github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.RetryStrategy) | optional | RetryStrategy is the strategy to retry a trigger if it fails | - + ### URLArtifact URLArtifact contains information about an artifact at an http endpoint. diff --git a/docs/trigger-guide.md b/docs/trigger-guide.md index 14688e9f6f..4bda075962 100644 --- a/docs/trigger-guide.md +++ b/docs/trigger-guide.md @@ -1,25 +1,12 @@ -# Trigger Guide +# Triggers Triggers are the sensor's actions. Triggers are only executed after all of the sensor's signals have been resolved. -The `resource` field in the trigger object has details of what to execute when the signals have been resolved. The `source` field in the `resource` object can have 3 types of values: - -- inline: -In this case, the workflow to execute as part of the trigger is inlined in the sensor yaml itself. E.g. [inline-sensor](https://github.com/argoproj/argo-events/blob/master/examples/sensors/inline-sensor.yaml) - -- file: -In this case, the workflow to execute is specified as a file-system path. This file-system path should exist in the sensor-controller deployment. The default sensor-controller does not have any volume mounts and therefore does not have any workflow yamls. If users are going to use this, they should explicitly mount appropriate volumes in the sensor-controller deployment. E.g. [file-sensor](https://github.com/argoproj/argo-events/blob/master/examples/sensors/file-sensor.yaml) - -- url: -In this case, the workflow to execute is specified as a url path. E.g. [url-sensor](https://github.com/argoproj/argo-events/blob/master/examples/sensors/url-sensor.yaml) - - +The `resource` field in the trigger object has details of what to execute when the signals have been resolved. Refer to https://github.com/argoproj/argo-events/blob/master/docs/artifact-guide.md ### Resource Object Resources define a YAML or JSON K8 resource. The set of currently resources supported are implemented in the `store` package. Adding support for new resources is as simple as including the type you want to create in the store's `decodeAndUnstructure()` method. We hope to change this functionality so that permissions for CRUD operations against certain resources can be controlled through RBAC roles instead. -List of currently supported K8 Resources: +List of currently supported custom K8 Resources: - Gateway - Sensor - [Workflow](https://github.com/argoproj/argo) -### Messages -Messages define content and a stream queue resource on which to send the content. \ No newline at end of file diff --git a/docs/tutorial.md b/docs/tutorial.md deleted file mode 100644 index fe3a9a5f67..0000000000 --- a/docs/tutorial.md +++ /dev/null @@ -1,1080 +0,0 @@ -# Guide - -1. [What are sensor and gateway controllers](controllers-guide.md) -2. [Learn about gateways](gateway-guide.md) -3. [Learn about sensors](sensor-guide.md) -4. [Learn about triggers](trigger-guide.md) -5. [Install gateways and sensors](#gands) - 1. [Webhook](#webhook) - 2. [Artifact](#artifact) - 3. [Calendar](#calendar) - 4. [Resource](#resource) - 5. [Streams](#streams) - 1. [Nats](#nats) - 2. [Kafka](#kafka) - 3. [MQTT](#mqtt) - 4. [AMQP](#amqp) -6. [Updating gateway configurations dynamically](#updating-configurations) -7. [Passing payload from signal to trigger](#passing-payload-from-signal-to-trigger) -8. [Sensor filters](#sensor-filters) -9. [Writing custom gateways](custom-gateway.md) - -## Install gateways and sensors - -## Webhook - -Webhook gateway is useful when you want to listen to an incoming HTTP request and forward that event to watchers. - -1)
Let's have a look at the configuration for our gateway.
- - ```yaml - apiVersion: v1 - kind: ConfigMap - metadata: - name: webhook-gateway-configmap - data: - # run http server on 12000 - webhook.portConfig: |- - port: "12000" - # listen to /bar endpoint for POST requests - webhook.barConfig: |- - endpoint: "/bar" - method: "POST" - # listen to /foo endpoint for POST requests - webhook.fooConfig: |- - endpoint: "/foo" - method: "POST" - ``` - - 1) This configmap contains multiple configurations. First configuration describes on which port HTTP server should run. Currently, the gateway - can only start one HTTP server and all endpoints will be registered with this server. But in future, we plan to add support to - spin up multiple HTTP servers and give ability to user to register endpoints to different servers. - - 2) Second configuration describes an endpoint called `/bar` that will be registered with HTTP server. The `method` describes which HTTP method - is allowed for a request. In this case only incoming HTTP POST requests will be accepted on `/bar`. - - 3) Third configuration has endpoint `/foo` and accepts requests with method POST. - -
Lets go ahead and create above configmap,
- - ```bash - kubectl create -n argo-events -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/webhook-gateway-configmap.yaml - ``` - - ```bash - # Make sure that configmap is created in `argo-events` namespace - - kubectl -n argo-events get configmaps webhook-gateway-configmap - ``` - -2)
Next step is to create the webhook gateway,
- - 1. Gateway definition, - ```yaml - apiVersion: argoproj.io/v1alpha1 - kind: Gateway - metadata: - # name of the gateway - name: webhook-gateway - labels: - # must match with instance id of one of the gateway controllers. - gateways.argoproj.io/gateway-controller-instanceid: argo-events - gateway-name: "webhook-gateway" - spec: - # configmap to read configurations from - configMap: "webhook-gateway-configmap" - # type of gateway - type: "webhook" - # event dispatch protocol between gateway and it's watchers - dispatchMechanism: "HTTP" - # version of events this gateway is generating. Required for cloudevents specification - version: "1.0" - # these are pod specifications - deploySpec: - containers: - - name: "webhook-events" - image: "argoproj/webhook-gateway" - imagePullPolicy: "Always" - command: ["/bin/webhook-gateway"] - serviceAccountName: "argo-events-sa" - # service specifications to expose gateway - serviceSpec: - selector: - gateway-name: "webhook-gateway" - ports: - - port: 12000 - targetPort: 12000 - type: LoadBalancer - # watchers are components interested in listening to events produced by this gateway - watchers: - sensors: - - name: "webhook-sensor" - ``` - - 2. Run following command, - ```bash - kubectl create -n argo-events -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/webhook.yaml - ``` - - 3. Check all gateway configurations are in `running` state - ```bash - kubectl get -n argo-events gateways webhook-gateway -o yaml - ``` - -3)
Now its time to create webhook sensor.
- 1. Sensor definition, - - ```yaml - apiVersion: argoproj.io/v1alpha1 - kind: Sensor - metadata: - # name of sensor - name: webhook-sensor - labels: - # instance-id must match with one of the deployed sensor controller's instance-id - sensors.argoproj.io/sensor-controller-instanceid: argo-events - spec: - # make this sensor as long running. - repeat: true - serviceAccountName: argo-events-sa - # signals/notifications this sensor is interested in. - signals: - # event must be from webhook-gateway and the configuration that produced this event must be - # webhook.fooConfig - - name: webhook-gateway/webhook.fooConfig - triggers: - - name: webhook-workflow-trigger - resource: - namespace: argo-events - group: argoproj.io - version: v1alpha1 - kind: Workflow - source: - inline: | - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: hello-world- - spec: - entrypoint: whalesay - templates: - - name: whalesay - container: - args: - - "hello world" - command: - - cowsay - image: "docker/whalesay:latest" - ``` - - This sensor defines only one signal called `webhook-gateway/webhook.fooConfig`, meaning, it is interested in listening - events from `webhook.fooConfig` configuration within `webhook-gateway` gateway. - - 2. Run following command, - ```bash - kubectl create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/sensors/webhook.yaml - ``` - - 3. Check whether all sensor nodes are initialized, - ```bash - kubectl get -n argo-events sensors webhook-sensor - ``` - - 4. Get the service url for gateway, - ```bash - minikube service --url webhook-gateway-gateway-svc - ``` - - 5. If you face issue getting service url from executing above command, you can use `kubectl port-forward` - 1. Open another terminal window and enter `kubectl port-forward 9003:` - 2. You can now user `localhost:9003` to query webhook gateway - - 6. Send a POST request to the gateway service, and monitor namespace for new workflow - ```bash - curl -d '{"message":"this is my first webhook"}' -H "Content-Type: application/json" -X POST /foo - ``` - - 7. List argo workflows - ```bash - argo -n argo-events list - ``` - -
- -## Artifact -Currently framework supports Minio S3 storage for artifact gateway but we plan to add File System and AWS/GCP S3 gateways in future. - -Lets start with deploying Minio server standalone deployment. You can get the K8 deployment from https://www.minio.io/kubernetes.html - - 1. Minio deployment, store it in `minio-deployment.yaml` - ```yaml - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - # This name uniquely identifies the PVC. Will be used in deployment below. - name: minio-pv-claim - labels: - app: minio-storage-claim - spec: - # Read more about access modes here: http://kubernetes.io/docs/user-guide/persistent-volumes/#access-modes - accessModes: - - ReadWriteOnce - resources: - # This is the request for storage. Should be available in the cluster. - requests: - storage: 10Gi - # Uncomment and add storageClass specific to your requirements below. Read more https://kubernetes.io/docs/concepts/storage/persistent-volumes/#class-1 - #storageClassName: - --- - apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - # This name uniquely identifies the Deployment - name: minio-deployment - spec: - strategy: - type: Recreate - template: - metadata: - labels: - # Label is used as selector in the service. - app: minio - spec: - # Refer to the PVC created earlier - volumes: - - name: storage - persistentVolumeClaim: - # Name of the PVC created earlier - claimName: minio-pv-claim - containers: - - name: minio - # Pulls the default Minio image from Docker Hub - image: minio/minio - args: - - server - - /storage - env: - # Minio access key and secret key - - name: MINIO_ACCESS_KEY - value: "myaccess" - - name: MINIO_SECRET_KEY - value: "mysecret" - ports: - - containerPort: 9000 - # Mount the volume into the pod - volumeMounts: - - name: storage # must match the volume name, above - mountPath: "/storage" - --- - apiVersion: v1 - kind: Service - metadata: - name: minio-service - spec: - type: LoadBalancer - ports: - - port: 9000 - targetPort: 9000 - protocol: TCP - selector: - app: minio - - ``` - - 2. Install minio, - ```bash - kubectl create -n argo-events -f minio-deployment.yaml - ``` - - 3. Create the configuration, - ```yaml - apiVersion: v1 - kind: ConfigMap - metadata: - name: artifact-gateway-configmap - data: - s3.fooConfig: |- - s3EventConfig: - bucket: input # name of the bucket we want to listen to - endpoint: minio-service.argo-events:9000 # minio service endpoint - event: s3:ObjectCreated:Put # type of event - filter: # filter on object name if any - prefix: "" - suffix: "" - insecure: true # type of minio server deployment - accessKey: - key: accesskey # key within below k8 secret whose corresponding value is name of the accessKey - name: artifacts-minio # k8 secret name that holds minio creds - secretKey: - key: secretkey # key within below k8 secret whose corresponding value is name of the secretKey - name: artifacts-minio # k8 secret name that holds minio creds - ``` - - Read comments on configmap to understand more about each field in configuration - - Run, - ```bash - kubectl create -n argo-events -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/artifact-gateway-configmap.yaml - ``` - - 4. Artifact gateway definition, - ```yaml - apiVersion: argoproj.io/v1alpha1 - kind: Gateway - metadata: - name: artifact-gateway - labels: - gateways.argoproj.io/gateway-controller-instanceid: argo-events - gateway-name: "artifact-gateway" - spec: - deploySpec: - containers: - - name: "artifact-events" - image: "argoproj/artifact-gateway" - imagePullPolicy: "Always" - command: ["/bin/artifact-gateway"] - serviceAccountName: "argo-events-sa" - configMap: "artifact-gateway-configmap" - version: "1.0" - type: "artifact" - dispatchMechanism: "HTTP" - watchers: - sensors: - - name: "artifact-sensor" - ``` - - Execute following command to create artifact gateway, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/artifact.yaml - ``` - - 5. Check whether all gateway configurations are active, - ```bash - kubectl -n argo-events get gateways artifact-gateway -o yaml - ``` - - 6. Below is the sensor definition, - ```yaml - apiVersion: argoproj.io/v1alpha1 - kind: Sensor - metadata: - name: artifact-sensor - labels: - sensors.argoproj.io/sensor-controller-instanceid: argo-events - spec: - repeat: true - serviceAccountName: argo-events-sa - signals: - - name: artifact-gateway/s3.fooConfig - triggers: - - name: artifact-workflow-trigger - resource: - namespace: argo-events - group: argoproj.io - version: v1alpha1 - kind: Workflow - source: - inline: | - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: hello-world- - spec: - entrypoint: whalesay - templates: - - - container: - args: - - "hello world" - command: - - cowsay - image: "docker/whalesay:latest" - name: whalesay - ``` - - Run, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/trigger-param-fix/examples/sensors/s3.yaml - ``` - - Check that all signals and triggers are intialized, - ```bash - kubectl -n argo-events get sensors artifact-sensor -o yaml - ``` - - 7. Drop a file into `input` bucket and monitor namespace for argo workflow. - ```bash - argo -n argo-events list - ``` - -
- -## Calendar -Calendar gateway either accepts `interval` or `cron schedules` as configuration. - - 1. Lets have a look at configuration, - ```bash - apiVersion: v1 - kind: ConfigMap - metadata: - name: calendar-gateway-configmap - data: - calendar.barConfig: |- - interval: 10s - calendar.fooConfig: |- - schedule: 30 * * * * - ``` - - The `barConfig` defines an interval of `10s`, meaning, gateway configuration will run every 10s and send event to watchers. - The `fooConfig` defines a cron schedule `30 * * * *` meaning, gateway configuration will run every 30 min and send event to watchers. - - Run, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/calendar-gateway-configmap.yaml - ``` - - 2. Gateway definition, - ```yaml - apiVersion: argoproj.io/v1alpha1 - kind: Gateway - metadata: - name: calendar-gateway - labels: - gateways.argoproj.io/gateway-controller-instanceid: argo-events - gateway-name: "calendar-gateway" - spec: - deploySpec: - containers: - - name: "calendar-events" - image: "argoproj/calendar-gateway" - imagePullPolicy: "Always" - command: ["/bin/calendar-gateway"] - serviceAccountName: "argo-events-sa" - configMap: "calendar-gateway-configmap" - type: "calendar" - dispatchMechanism: "HTTP" - version: "1.0" - watchers: - sensors: - - name: "calendar-sensor" - ``` - - Run, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/calendar.yaml - ``` - - Check all configurations are active, - ```bash - kubectl -n argo-events get gateways calendar-gateway -o yaml - ``` - - 3. Sensor definition, - ```yaml - apiVersion: argoproj.io/v1alpha1 - kind: Sensor - metadata: - name: calendar-sensor - labels: - sensors.argoproj.io/sensor-controller-instanceid: argo-events - spec: - serviceAccountName: argo-events-sa - imagePullPolicy: Always - repeat: true - signals: - - name: calendar-gateway/calendar.fooConfig - triggers: - - name: calendar-workflow-trigger - resource: - namespace: argo-events - group: argoproj.io - version: v1alpha1 - kind: Workflow - source: - inline: | - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: hello-world- - spec: - entrypoint: whalesay - templates: - - - container: - args: - - "hello world" - command: - - cowsay - image: "docker/whalesay:latest" - name: whalesay - ``` - - Run, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/sensors/calendar.yaml - ``` - - 4. List workflows, - ```bash - argo -n argo-events list - ``` - -
- -## Resource -Resource gateway can monitor any K8 resource and any CRD. - - 1. Lets have a look at a configuration, - ```yaml - apiVersion: v1 - kind: ConfigMap - metadata: - name: resource-gateway-configmap - data: - resource.fooConfig: |- - namespace: argo-events - group: "argoproj.io" - version: "v1alpha1" - kind: "Workflow" - filter: - labels: - workflows.argoproj.io/phase: Succeeded - name: "my-workflow" - resource.barConfig: |- - namespace: argo-events - group: "argoproj.io" - version: "v1alpha1" - kind: "Workflow" - filter: - prefix: scripts-bash - labels: - workflows.argoproj.io/phase: Failed - ``` - - * In configuration `resource.fooConfig`, gateway will watch resource of type `Workflow` which is K8 CRD. Whenever a - workflow with name `my-workflow` is assigned label `workflows.argoproj.io/phase: Succeeded`, the configuration will - send an event to watchers. - - * Gateway configuration `resource.barConfig` will send event to watchers whenever a sensor label `workflows.argoproj.io/phase: Failed` is added. - - * You can create more such configurations that watch namespace, configmaps, deployments, pods etc. - - Run, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/resource-gateway-configmap.yaml - ``` - - 2. Gateway definition, - ```yaml - apiVersion: argoproj.io/v1alpha1 - kind: Gateway - metadata: - name: resource-gateway - labels: - gateways.argoproj.io/gateway-controller-instanceid: argo-events - gateway-name: "resource-gateway" - spec: - deploySpec: - containers: - - name: "resource-events" - image: "argoproj/resource-gateway" - imagePullPolicy: "Always" - command: ["/bin/resource-gateway"] - serviceAccountName: "argo-events-sa" - configMap: "resource-gateway-configmap" - type: "resource" - dispatchMechanism: "HTTP" - version: "1.0" - watchers: - sensors: - - name: "resource-sensor" - ``` - - Run, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/resource.yaml - ``` - - Check all configurations are active, - ```bash - kubectl -n argo-events get gateways resource-gateway -o yaml - ``` - - 3. Sensor definition, - ```yaml - apiVersion: argoproj.io/v1alpha1 - kind: Sensor - metadata: - name: resource-sensor - labels: - sensors.argoproj.io/sensor-controller-instanceid: argo-events - spec: - repeat: true - serviceAccountName: argo-events-sa - signals: - - name: resource-gateway/resource.fooConfig - triggers: - - name: argo-workflow - resource: - namespace: argo-events - group: argoproj.io - version: v1alpha1 - kind: Workflow - source: - inline: | - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: hello-world- - spec: - entrypoint: whalesay - templates: - - - container: - args: - - "hello world" - command: - - cowsay - image: "docker/whalesay:latest" - name: whalesay - - ``` - - Run, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/sensors/resource.yaml - ``` - - 4. Create an basic `hello-world` argo workflow with name `my-workflow`. - ```yaml - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: hello-world- - namespace: argo-events - labels: - name: my-workflow - spec: - entrypoint: whalesay - serviceAccountName: argo-events-sa - templates: - - container: - args: - - "hello world" - command: - - cowsay - image: docker/whalesay:latest - name: whalesay - ``` - Run - ```bash - kubectl -n argo-events -f https://raw.githubusercontent.com/argoproj/argo/master/examples/hello-world.yaml - ``` - - Once workflow is created, resource sensor will trigger workflow. - - 5. Run `argo -n argo-events list` - -
- -## Streams - * ### NATS - Lets start by installing a NATS cluster - - 1) Store following NATS deployment in nats-deploy.yaml - ```yaml - apiVersion: v1 - kind: Service - metadata: - name: nats - namespace: argo-events - labels: - component: nats - spec: - selector: - component: nats - type: ClusterIP - ports: - - name: client - port: 4222 - - name: cluster - port: 6222 - - name: monitor - port: 8222 - --- - apiVersion: apps/v1beta1 - kind: StatefulSet - metadata: - name: nats - namespace: argo-events - labels: - component: nats - spec: - serviceName: nats - replicas: 1 - template: - metadata: - labels: - component: nats - spec: - serviceAccountName: argo-events-sa - containers: - - name: nats - image: nats:latest - ports: - - containerPort: 4222 - name: client - - containerPort: 6222 - name: cluster - - containerPort: 8222 - name: monitor - livenessProbe: - httpGet: - path: / - port: 8222 - initialDelaySeconds: 10 - timeoutSeconds: 5 - ``` - Run, - ```bash - kubectl -n argo-events create -f nats-deploy.yaml - ``` - - 2) Once all pods are up and running, create gateway configmap, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/nats-gateway-configmap.yaml - ``` - - 3) Lets create a sensor, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/sensors/nats.yaml - ``` - - 4) Lets create gateway, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/nats.yaml - ``` - 5) Use nats client to publish message to subject. To install NATS client, head to [go-nats](https://github.com/nats-io/go-nats) - - 6) Once you publish message to a subject the gateway is configured to listen, you will see the argo workflow getting created. - - * ### Kafka - 1) If you don't already have a Kafka cluster running, follow the [kafka setup](https://github.com/helm/charts/tree/master/incubator/kafka) - - 2) Lets create the configuration for gateway, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/kafka-gateway-configmap.yaml - ``` - - 3) Once above configmap is created, lets deploy the gateway, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/kafka.yaml - ``` - - 4) To create sensor, run - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/sensors/kafka.yaml - ``` - - 5) Publish a message to a topic and partition the gateway is configured to listen, you will see the argo workflow getting created. - - * ### MQTT - 1) If you don't have MQTT broker installed, use `Mosquitto`. - - Deployment, - ```yaml - apiVersion: extensions/v1beta1 - kind: Deployment - metadata: - name: mosquitto - namespace: argo-events - spec: - template: - spec: - serviceAccountName: argo-events-sa - containers: - - name: mosquitto - image: toke/mosquitto - ports: - - containerPort: 9001 - - containerPort: 8883 - ``` - - Service, - ```yaml - apiVersion: v1 - kind: Service - metadata: - name: mqtt - namespace: argo-events - spec: - ports: - - name: mosquitto - port: 1883 - - name: mosquitto-web - port: 80 - targetPort: 9001 - ``` - - 2. Create the gateway configuration, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/mqtt-gateway-configmap.yaml - ``` - - 3. Deploy the gateway, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateway/mqtt-gateway.yaml - ``` - - 4. Deploy the sensor, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/sensors/mqtt-sensor.yaml - ``` - - 5. Send a message to correct topic the gateway is configured to listen, you will see the argo workflow getting created. - - * ### AMQP - 1) If you haven't already setup rabbitmq cluster, follow [rabbitmq setup](https://github.com/binarin/rabbit-on-k8s-standalone) - - 2) Create gateway configuration, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/amqp-gateway-configmap.yaml - ``` - - 3) Deploy gateway, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/gateways/amqp.yaml - ``` - - 4) Deploy sensor, - ```bash - kubectl -n argo-events create -f https://raw.githubusercontent.com/argoproj/argo-events/master/examples/sensors/amqp.yaml - ``` - - 5) Send a message to exchange name the gateway is configured to listen, you will see the argo workflow getting created. - - -
- -## Updating gateway configurations dynamically - The framework offers ability to add and remove configurations for gateway on the fly. - Lets look at an example of webhook gateway. You already have three configurations running in gateway, - `webhook.portConfig`, `webhook.fooConfig` and `webhook.barConfig` - - 1) Lets add a new configuration to gateway configmap. Update configmap looks like, - ```yaml - apiVersion: v1 - kind: ConfigMap - metadata: - name: webhook-gateway-configmap - data: - webhook.portConfig: |- - port: "12000" - webhook.barConfig: |- - endpoint: "/bar" - method: "POST" - webhook.fooConfig: |- - endpoint: "/foo" - method: "POST" - webhook.myNewConfig: |- - endpoint: "/my" - method: "POST" - ``` - Run `kubectl -n argo-events apply -f configmap-file-name` on gateway configmap to update the configmap resource. - - 2) Run `kubectl -n argo-events get gateways webhook-gateway -o yaml`, you'll see gateway now has `webhook.myNewConfig` running. - - 3) Update the webhook sensor or create a new sensor to listen to this new configuration. - - 4) Test the endpoint by firing a HTTP POST request to `/my`. - - 5) Now, lets remove the configuration `webhook.myNewConfig` from gateway configmap. Run `kubectl apply` to update the configmap. - - 6) Check the gateway resource, `kubectl -n argo-events get gateways webhook-gateway -o yaml`. You will see `webhook.myNewConfig` is removed from the gateway. - - 7) Try sending a POST request to '/my' and server will respond with 404. - -
- -## Passing payload from signal to trigger - - * ### Complete payload - - 1. Create a webhook sensor, - ```yaml - apiVersion: argoproj.io/v1alpha1 - kind: Sensor - metadata: - name: webhook-with-resource-param-sensor - labels: - sensors.argoproj.io/sensor-controller-instanceid: argo-events - spec: - repeat: true - serviceAccountName: argo-events-sa - signals: - - name: webhook-gateway/webhook.fooConfig - triggers: - - name: argo-workflow - resource: - namespace: argo-events - group: argoproj.io - version: v1alpha1 - kind: Workflow - parameters: - - src: - signal: webhook-gateway/webhook.fooConfig - # pass payload of webhook-gateway/webhook.fooConfig signal to first parameter value - # of arguments. - dest: spec.arguments.parameters.0.value - source: - inline: | - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - name: arguments-via-webhook-event - spec: - entrypoint: whalesay - arguments: - parameters: - - name: message - # this is the value that should be overridden - value: hello world - templates: - - name: whalesay - inputs: - parameters: - - name: message - container: - image: docker/whalesay:latest - command: [cowsay] - args: ["{{inputs.parameters.message}}"] - ``` - - Run, - ```bash - kubectl create -f https://raw.githubusercontent.com/argoproj/argo-events/trigger-param-fix/examples/sensors/webhook-with-complete-payload.yaml - ``` - - 2. Note that sensor name is `webhook-with-resource-param-sensor`. Update your gateway accordingly or create a new one. - - 3. Send a POST request to your webhook gateway - ```bash - curl -d '{"message":"this is my first webhook"}' -H "Content-Type: application/json" -X POST $WEBHOOK_SERVICE_URL/foo - ``` - - 4. List argo workflows, - ```bash - argo -n argo-events list - ``` - - 5. Check the workflow logs using `argo -n argo-events logs ` - - - ## Filter event payload - 1. Create a webhook sensor, - ```yaml - apiVersion: argoproj.io/v1alpha1 - kind: Sensor - metadata: - name: webhook-with-resource-param-sensor - labels: - sensors.argoproj.io/sensor-controller-instanceid: argo-events - spec: - repeat: true - serviceAccountName: argo-events-sa - signals: - - name: webhook-gateway/webhook.fooConfig - triggers: - - name: argo-workflow - resource: - namespace: argo-events - group: argoproj.io - version: v1alpha1 - kind: Workflow - # The parameters from the workflow are overridden by the webhook's message - parameters: - - src: - signal: webhook-gateway/webhook.fooConfig - # extract the object corresponding to `message` key from event payload - # of webhook-gateway/webhook.fooConfig signal - path: message - # if `message` key doesn't exists in event payload then default value of payload - # passed to trigger will be `hello default` - value: hello default - # override the value of first parameter in arguments with above payload. - dest: spec.arguments.parameters.0.value - source: - inline: | - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - name: arguments-via-webhook-event - spec: - entrypoint: whalesay - arguments: - parameters: - - name: message - # this is the value that should be overridden - value: hello world - templates: - - name: whalesay - inputs: - parameters: - - name: message - container: - image: docker/whalesay:latest - command: [cowsay] - args: ["{{inputs.parameters.message}}"] - - ``` - 2. Run, - ```bash - kubectl apply -f https://raw.githubusercontent.com/argoproj/argo-events/trigger-param-fix/examples/sensors/webhook-with-resource-param.yaml - ``` - - 3. Post request to webhook gateway and watch new workflow being created - -
- -## Sensor Filters - Following are the types of the filter you can apply on signal/event payload, - - | Type | Description | - |----------|-------------------| - | Time | Filters the signal based on time constraints | - | EventContext | Filters metadata that provides circumstantial information about the signal. | - | Data | Describes constraints and filters for payload | - - ### Time Filter - ```yaml - filters: - time: - start: "2016-05-10T15:04:05Z07:00" - stop: "2020-01-02T15:04:05Z07:00" - ``` - - Example: - https://raw.githubusercontent.com/argoproj/argo-events/master/examples/sensors/time-filter-webhook.yaml - - ### EventContext Filter - ``` - filters: - context: - source: - host: amazon.com - contentType: application/json - ``` - - Example: - https://raw.githubusercontent.com/argoproj/argo-events/master/examples/sensors/context-filter-webhook.yaml - - ### Data filter - ``` - filters: - data: - - path: bucket - type: string - value: argo-workflow-input - ``` - Example: - https://raw.githubusercontent.com/argoproj/argo-events/master/examples/sensors/data-filter-webhook.yaml diff --git a/examples/gateways/amqp.yaml b/examples/gateways/amqp.yaml index 6ab7db9b16..e6c767a338 100644 --- a/examples/gateways/amqp.yaml +++ b/examples/gateways/amqp.yaml @@ -6,18 +6,30 @@ metadata: gateways.argoproj.io/gateway-controller-instanceid: argo-events gateway-name: "amqp-gateway" spec: + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" deploySpec: - containers: - - name: amqp-events - image: "argoproj/amqp-gateway" - imagePullPolicy: "Always" - command: ["/bin/amqp-gateway"] - serviceAccountName: "argo-events-sa" + metadata: + name: "amqp-gateway" + labels: + gateway-name: "amqp-gateway" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: amqp-events + image: "argoproj/amqp-gateway" + imagePullPolicy: "Always" + command: ["/bin/amqp-gateway"] + serviceAccountName: "argo-events-sa" configMap: "amqp-gateway-configmap" type: "amqp" - dispatchMechanism: "HTTP" eventVersion: "1.0" - imageVersion: "latest" watchers: sensors: - name: "amqp-sensor" diff --git a/examples/gateways/artifact-gateway-configmap.yaml b/examples/gateways/artifact-gateway-configmap.yaml index da5a3f5fa4..264148fada 100644 --- a/examples/gateways/artifact-gateway-configmap.yaml +++ b/examples/gateways/artifact-gateway-configmap.yaml @@ -4,13 +4,13 @@ metadata: name: artifact-gateway-configmap data: input: |- - s3EventConfig: - bucket: input - endpoint: minio-service.argo-events:9000 - event: s3:ObjectCreated:Put - filter: - prefix: "" - suffix: "" + bucket: + name: input + endpoint: minio-service.argo-events:9000 + event: s3:ObjectCreated:Put + filter: + prefix: "" + suffix: "" insecure: true accessKey: key: accesskey diff --git a/examples/gateways/artifact-http.yaml b/examples/gateways/artifact-http.yaml new file mode 100644 index 0000000000..dae3234937 --- /dev/null +++ b/examples/gateways/artifact-http.yaml @@ -0,0 +1,36 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Gateway +metadata: + name: artifact-gateway + labels: + gateways.argoproj.io/gateway-controller-instanceid: argo-events + gateway-name: "artifact-gateway" +spec: + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" + deploySpec: + metadata: + name: "artifact-gateway" + labels: + gateway-name: "artifact-gateway" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "artifact-events" + image: "argoproj/artifact-gateway" + imagePullPolicy: "Always" + command: ["/bin/artifact-gateway"] + serviceAccountName: "argo-events-sa" + configMap: "artifact-gateway-configmap" + eventVersion: "1.0" + type: "artifact" + watchers: + sensors: + - name: "artifact-sensor" + - name: "artifact-with-param-sensor" diff --git a/examples/gateways/artifact-nats-standard.yaml b/examples/gateways/artifact-nats-standard.yaml new file mode 100644 index 0000000000..94c9401cba --- /dev/null +++ b/examples/gateways/artifact-nats-standard.yaml @@ -0,0 +1,33 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Gateway +metadata: + name: artifact-gateway-nats-standard + labels: + gateways.argoproj.io/gateway-controller-instanceid: argo-events + gateway-name: "artifact-gateway-nats-standard" +spec: + processorPort: "9330" + eventProtocol: + type: "NATS" + nats: + url: "nats://example-nats.argo-events:4222" + type: "Standard" + deploySpec: + metadata: + name: "artifact-gateway-nats-standard" + labels: + gateway-name: "artifact-gateway-nats-standard" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "artifact-events" + image: "argoproj/artifact-gateway" + imagePullPolicy: "Always" + command: ["/bin/artifact-gateway"] + serviceAccountName: "argo-events-sa" + configMap: "artifact-gateway-configmap" + eventVersion: "1.0" + type: "artifact" diff --git a/examples/gateways/artifact-nats-streaming.yaml b/examples/gateways/artifact-nats-streaming.yaml new file mode 100644 index 0000000000..b274cca678 --- /dev/null +++ b/examples/gateways/artifact-nats-streaming.yaml @@ -0,0 +1,35 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Gateway +metadata: + name: artifact-gateway-nats-streaming + labels: + gateways.argoproj.io/gateway-controller-instanceid: argo-events + gateway-name: "artifact-gateway-nats-streaming" +spec: + processorPort: "9330" + eventProtocol: + type: "NATS" + nats: + url: "nats://example-nats.argo-events:4222" + clusterId: "example-stan" + clientId: "myclient" + type: "Streaming" + deploySpec: + metadata: + name: "artifact-gateway-nats-streaming" + labels: + gateway-name: "artifact-gateway-nats-streaming" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "artifact-events" + image: "argoproj/artifact-gateway" + imagePullPolicy: "Always" + command: ["/bin/artifact-gateway"] + serviceAccountName: "argo-events-sa" + configMap: "artifact-gateway-configmap" + eventVersion: "1.0" + type: "artifact" diff --git a/examples/gateways/artifact.yaml b/examples/gateways/artifact.yaml deleted file mode 100644 index f5acf5d0d5..0000000000 --- a/examples/gateways/artifact.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Gateway -metadata: - name: artifact-gateway - labels: - gateways.argoproj.io/gateway-controller-instanceid: argo-events - gateway-name: "artifact-gateway" -spec: - deploySpec: - containers: - - name: "artifact-events" - image: "argoproj/artifact-gateway" - imagePullPolicy: "Always" - command: ["/bin/artifact-gateway"] - serviceAccountName: "argo-events-sa" - configMap: "artifact-gateway-configmap" - eventVersion: "1.0" - imageVersion: "latest" - type: "artifact" - dispatchMechanism: "HTTP" - watchers: - sensors: - - name: "artifact-sensor" diff --git a/examples/gateways/calendar-gateway-configmap.yaml b/examples/gateways/calendar-gateway-configmap.yaml index 0683c65718..80467e80d4 100644 --- a/examples/gateways/calendar-gateway-configmap.yaml +++ b/examples/gateways/calendar-gateway-configmap.yaml @@ -3,7 +3,7 @@ kind: ConfigMap metadata: name: calendar-gateway-configmap data: - foo: |- + interval: |- interval: 55s - bar: |- + schedule: |- schedule: 30 * * * * diff --git a/examples/gateways/calendar.yaml b/examples/gateways/calendar.yaml index 3674e5797f..e3bba9c54a 100644 --- a/examples/gateways/calendar.yaml +++ b/examples/gateways/calendar.yaml @@ -6,18 +6,30 @@ metadata: gateways.argoproj.io/gateway-controller-instanceid: argo-events gateway-name: "calendar-gateway" spec: + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" deploySpec: - containers: - - name: "calendar-events" - image: "argoproj/calendar-gateway" - imagePullPolicy: "Always" - command: ["/bin/calendar-gateway"] - serviceAccountName: "argo-events-sa" + metadata: + name: "calendar-gateway" + labels: + gateway-name: "calendar-gateway" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "calendar-events" + image: "argoproj/calendar-gateway" + imagePullPolicy: "Always" + command: ["/bin/calendar-gateway"] + serviceAccountName: "argo-events-sa" configMap: "calendar-gateway-configmap" type: "calendar" - dispatchMechanism: "HTTP" eventVersion: "1.0" - imageVersion: "latest" watchers: sensors: - name: "calendar-sensor" diff --git a/examples/gateways/file.yaml b/examples/gateways/file.yaml index c7b04d16fe..5255523ea1 100644 --- a/examples/gateways/file.yaml +++ b/examples/gateways/file.yaml @@ -6,18 +6,30 @@ metadata: gateways.argoproj.io/gateway-controller-instanceid: argo-events gateway-name: "file-gateway" spec: + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" deploySpec: - containers: - - name: "file-events" - image: "argoproj/file-gateway" - imagePullPolicy: "Always" - command: ["/bin/file-gateway"] - serviceAccountName: "argo-events-sa" + metadata: + name: "file-gateway" + labels: + gateway-name: "file-gateway" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "file-events" + image: "argoproj/file-gateway" + imagePullPolicy: "Always" + command: ["/bin/file-gateway"] + serviceAccountName: "argo-events-sa" configMap: "file-gateway-configmap" eventVersion: "1.0" - imageVersion: "latest" type: "file" - dispatchMechanism: "HTTP" watchers: sensors: - name: "file-sensor" diff --git a/examples/gateways/github-configmap.yaml b/examples/gateways/github-configmap.yaml index 6004bcf30f..f9edcb656c 100644 --- a/examples/gateways/github-configmap.yaml +++ b/examples/gateways/github-configmap.yaml @@ -6,7 +6,7 @@ data: project_1: |- owner: "owner-example" repository: "repo-example" - url: "http://webhook-gateway-gateway-svc/push" + url: "http://webhook-gateway-http-svc/push" events: - "*" apiToken: diff --git a/examples/gateways/github.yaml b/examples/gateways/github.yaml index 4c4069130b..b47cc52a02 100644 --- a/examples/gateways/github.yaml +++ b/examples/gateways/github.yaml @@ -6,27 +6,30 @@ metadata: gateways.argoproj.io/gateway-controller-instanceid: argo-events gateway-name: "github-gateway" spec: + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" deploySpec: metadata: + name: "github-gateway" labels: - gateway-type: github - dispatch-mechanism: http + gateway-name: "github-gateway" spec: containers: - name: "gateway-client" - image: "metalgearsolid/gateway-client:v0.6.1" + image: "argoproj/gateway-client" imagePullPolicy: "Always" command: ["/bin/gateway-client"] - - name: "file-events" - image: "metalgearsolid/github-gateway:v0.6.1" + - name: "github-events" + image: "argoproj/github-gateway" imagePullPolicy: "Always" command: ["/bin/github-gateway"] serviceAccountName: "argo-events-sa" configMap: "github-gateway-configmap" type: "github" - dispatchProtocol: "HTTP" eventVersion: "1.0" - processorPort: "9330" watchers: sensors: - name: "github-sensor" diff --git a/examples/gateways/gitlab.yaml b/examples/gateways/gitlab.yaml index 47bf974dcd..5be3fa426c 100644 --- a/examples/gateways/gitlab.yaml +++ b/examples/gateways/gitlab.yaml @@ -6,18 +6,30 @@ metadata: gateways.argoproj.io/gateway-controller-instanceid: argo-events gateway-name: "gitlab-gateway" spec: + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" deploySpec: - containers: - - name: "gitlab-events" - image: "argoproj/gitlab-gateway" - imagePullPolicy: "Always" - command: ["/bin/gitlab-gateway"] - serviceAccountName: "argo-events-sa" + metadata: + name: "gitlab-gateway" + labels: + gateway-name: "gitlab-gateway" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "gitlab-events" + image: "argoproj/gitlab-gateway" + imagePullPolicy: "Always" + command: ["/bin/gitlab-gateway"] + serviceAccountName: "argo-events-sa" configMap: "gitlab-gateway-configmap" type: "gitlab" - dispatchMechanism: "HTTP" eventVersion: "1.0" - imageVersion: "latest" watchers: sensors: - name: "gitlab-sensor" diff --git a/examples/gateways/kafka.yaml b/examples/gateways/kafka.yaml index f4287edfcd..449682817e 100644 --- a/examples/gateways/kafka.yaml +++ b/examples/gateways/kafka.yaml @@ -6,18 +6,30 @@ metadata: gateways.argoproj.io/gateway-controller-instanceid: argo-events gateway-name: "kafka-gateway" spec: + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" deploySpec: - containers: - - name: "kafka-events" - image: "argoproj/kafka-gateway" - imagePullPolicy: "Always" - command: ["/bin/kafka-gateway"] - serviceAccountName: "argo-events-sa" + metadata: + name: "kafka-gateway" + labels: + gateway-name: "kafka-gateway" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "kafka-events" + image: "argoproj/kafka-gateway" + imagePullPolicy: "Always" + command: ["/bin/kafka-gateway"] + serviceAccountName: "argo-events-sa" configMap: "kafka-gateway-configmap" type: "kafka" - dispatchMechanism: "HTTP" eventVersion: "1.0" - imageVersion: "latest" watchers: sensors: - name: "kafka-sensor" diff --git a/examples/gateways/mqtt.yaml b/examples/gateways/mqtt.yaml index 6f0fbb1b56..8d15408e15 100644 --- a/examples/gateways/mqtt.yaml +++ b/examples/gateways/mqtt.yaml @@ -6,18 +6,30 @@ metadata: gateways.argoproj.io/gateway-controller-instanceid: argo-events gateway-name: "mqtt-gateway" spec: + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" deploySpec: - containers: - - name: "mqtt-events" - image: "argoproj/mqtt-gateway" - imagePullPolicy: "Always" - command: ["/bin/mqtt-gateway"] - serviceAccountName: "argo-events-sa" + metadata: + name: "mqtt-gateway" + labels: + gateway-name: "mqtt-gateway" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "mqtt-events" + image: "argoproj/mqtt-gateway" + imagePullPolicy: "Always" + command: ["/bin/mqtt-gateway"] + serviceAccountName: "argo-events-sa" configMap: "mqtt-gateway-configmap" type: "mqtt" - dispatchMechanism: "HTTP" eventVersion: "1.0" - imageVersion: "latest" watchers: sensors: - - name: "mqtt-sensor" + - name: "mqtt-sensor" diff --git a/examples/gateways/multi-watchers.yaml b/examples/gateways/multi-watchers.yaml index fd82ee0da7..1132ac8758 100644 --- a/examples/gateways/multi-watchers.yaml +++ b/examples/gateways/multi-watchers.yaml @@ -6,25 +6,40 @@ metadata: gateways.argoproj.io/gateway-controller-instanceid: argo-events gateway-name: "webhook-gateway-multi-watchers" spec: + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" + deploySpec: + metadata: + name: "webhook-gateway" + labels: + gateway-name: "webhook-gateway" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "webhook-events" + image: "argoproj/webhook-gateway" + imagePullPolicy: "Always" + command: ["/bin/webhook-gateway"] + serviceAccountName: "argo-events-sa" configMap: "webhook-gateway-configmap" type: "webhook" - dispatchMechanism: "HTTP" eventVersion: "1.0" - imageVersion: "latest" - deploySpec: - containers: - - name: "webhook-events" - image: "argoproj/webhook-gateway" - imagePullPolicy: "Always" - command: ["/bin/webhook-gateway"] - serviceAccountName: "argo-events-sa" serviceSpec: - selector: - gateway-name: "webhook-gateway-multi-watchers" - ports: - - port: 12000 - targetPort: 12000 - type: LoadBalancer + metadata: + name: webhook-gateway-svc + spec: + selector: + gateway-name: "webhook-gateway-http" + ports: + - port: 12000 + targetPort: 12000 + type: LoadBalancer watchers: # requirement for adding gateway as watcher is there should be a http server running in watcher gateway # and user must provide port and endpoint on which event should be dispatched. diff --git a/examples/gateways/nats.yaml b/examples/gateways/nats.yaml index 0be0af563a..67876d02a3 100644 --- a/examples/gateways/nats.yaml +++ b/examples/gateways/nats.yaml @@ -6,18 +6,30 @@ metadata: gateways.argoproj.io/gateway-controller-instanceid: argo-events gateway-name: "nats-gateway" spec: + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" deploySpec: - containers: - - name: "nats-events" - image: "argoproj/nats-gateway" - imagePullPolicy: "Always" - command: ["/bin/nats-gateway"] - serviceAccountName: "argo-events-sa" + metadata: + name: "nats-gateway" + labels: + gateway-name: "nats-gateway" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "nats-events" + image: "argoproj/nats-gateway" + imagePullPolicy: "Always" + command: ["/bin/nats-gateway"] + serviceAccountName: "argo-events-sa" configMap: "nats-gateway-configmap" type: "nats" - dispatchMechanism: "HTTP" eventVersion: "1.0" - imageVersion: "latest" watchers: sensors: - name: "nats-sensor" diff --git a/examples/gateways/resource.yaml b/examples/gateways/resource.yaml index eaae5e114e..1e208527a6 100644 --- a/examples/gateways/resource.yaml +++ b/examples/gateways/resource.yaml @@ -1,4 +1,3 @@ - apiVersion: argoproj.io/v1alpha1 kind: Gateway metadata: @@ -7,18 +6,30 @@ metadata: gateways.argoproj.io/gateway-controller-instanceid: argo-events gateway-name: "resource-gateway" spec: + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" deploySpec: - containers: - - name: "resource-events" - image: "argoproj/resource-gateway" - imagePullPolicy: "Always" - command: ["/bin/resource-gateway"] - serviceAccountName: "argo-events-sa" + metadata: + name: "resource-gateway" + labels: + gateway-name: "resource-gateway" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "resource-events" + image: "argoproj/resource-gateway" + imagePullPolicy: "Always" + command: ["/bin/resource-gateway"] + serviceAccountName: "argo-events-sa" configMap: "resource-gateway-configmap" type: "resource" - dispatchMechanism: "HTTP" eventVersion: "1.0" - imageVersion: "latest" watchers: sensors: - name: "resource-sensor" diff --git a/examples/gateways/storage-grid-configmap.yaml b/examples/gateways/storage-grid-configmap.yaml index e6807c3191..99e1e0b751 100644 --- a/examples/gateways/storage-grid-configmap.yaml +++ b/examples/gateways/storage-grid-configmap.yaml @@ -8,6 +8,3 @@ data: endpoint: "/" events: - "ObjectCreated:Put" - filter: - suffix: ".txt" - prefix: "hello-" diff --git a/examples/gateways/storage-grid.yaml b/examples/gateways/storage-grid.yaml index 74c4fb29d8..a643e1aca4 100644 --- a/examples/gateways/storage-grid.yaml +++ b/examples/gateways/storage-grid.yaml @@ -8,22 +8,37 @@ metadata: spec: configMap: "storage-grid-gateway-configmap" type: "storage_grid" - dispatchMechanism: "HTTP" + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" eventVersion: "1.0" - imageVersion: "latest" deploySpec: - containers: - - name: "storage-grid-events" - image: "argoproj/storage-grid-gateway" - imagePullPolicy: "Always" - serviceAccountName: "argo-events-sa" + metadata: + name: "storage-grid-gateway" + labels: + gateway-name: "storage-grid-gateway" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "storage-grid-events" + image: "argoproj/storage-grid-gateway" + imagePullPolicy: "Always" + serviceAccountName: "argo-events-sa" serviceSpec: - selector: - gateway-name: "storage-grid-gateway" - ports: - - port: 8080 - targetPort: 8080 - type: LoadBalancer + metadata: + name: storage-grid-gateway-svc + spec: + selector: + gateway-name: "storage-grid-gateway" + ports: + - port: 8080 + targetPort: 8080 + type: LoadBalancer watchers: sensors: - name: "storage-grid-watcher-sensor" diff --git a/examples/gateways/webhook-gateway-configmap.yaml b/examples/gateways/webhook-gateway-configmap.yaml index d4a266d74b..e074b3011e 100644 --- a/examples/gateways/webhook-gateway-configmap.yaml +++ b/examples/gateways/webhook-gateway-configmap.yaml @@ -5,8 +5,9 @@ metadata: data: index: |- port: "12000" - endpoint: "/" + endpoint: "/index" method: "POST" foo: |- + port: "12000" endpoint: "/foo" method: "POST" diff --git a/examples/gateways/webhook-http.yaml b/examples/gateways/webhook-http.yaml new file mode 100644 index 0000000000..bf431d5879 --- /dev/null +++ b/examples/gateways/webhook-http.yaml @@ -0,0 +1,45 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Gateway +metadata: + name: webhook-gateway-http + labels: + gateways.argoproj.io/gateway-controller-instanceid: argo-events + gateway-name: "webhook-gateway-http" +spec: + configMap: "webhook-gateway-configmap" + type: "webhook" + processorPort: "9330" + eventProtocol: + type: "HTTP" + http: + port: "9300" + eventVersion: "1.0" + deploySpec: + metadata: + name: "webhook-gateway-http" + labels: + gateway-name: "webhook-gateway-http" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "webhook-events" + image: "argoproj/webhook-gateway" + imagePullPolicy: "Always" + command: ["/bin/webhook-gateway"] + serviceAccountName: "argo-events-sa" + serviceSpec: + metadata: + name: webhook-gateway-svc + spec: + selector: + gateway-name: "webhook-gateway-http" + ports: + - port: 12000 + targetPort: 12000 + type: LoadBalancer + watchers: + sensors: + - name: "webhook-sensor-http" diff --git a/examples/gateways/webhook-nats-standard.yaml b/examples/gateways/webhook-nats-standard.yaml new file mode 100644 index 0000000000..04a2aed176 --- /dev/null +++ b/examples/gateways/webhook-nats-standard.yaml @@ -0,0 +1,43 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Gateway +metadata: + name: webhook-gateway-nats + labels: + gateways.argoproj.io/gateway-controller-instanceid: argo-events + gateway-name: "webhook-gateway-nats" +spec: + configMap: "webhook-gateway-configmap" + type: "webhook" + processorPort: "9330" + eventProtocol: + type: "NATS" + nats: + url: "nats://nats.argo-events:4222" + type: "Standard" + eventVersion: "1.0" + deploySpec: + metadata: + name: "webhook-gateway-nats" + labels: + gateway-name: "webhook-gateway-nats" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "webhook-events" + image: "argoproj/webhook-gateway" + imagePullPolicy: "Always" + command: ["/bin/webhook-gateway"] + serviceAccountName: "argo-events-sa" + serviceSpec: + metadata: + name: webhook-gateway-svc + spec: + selector: + gateway-name: "webhook-gateway-nats" + ports: + - port: 12000 + targetPort: 12000 + type: LoadBalancer diff --git a/examples/gateways/webhook-nats-streaming.yaml b/examples/gateways/webhook-nats-streaming.yaml new file mode 100644 index 0000000000..02dfa527e7 --- /dev/null +++ b/examples/gateways/webhook-nats-streaming.yaml @@ -0,0 +1,45 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Gateway +metadata: + name: webhook-gateway-nats-streaming + labels: + gateways.argoproj.io/gateway-controller-instanceid: argo-events + gateway-name: "webhook-gateway-nats-streaming" +spec: + configMap: "webhook-gateway-configmap" + type: "webhook" + processorPort: "9330" + eventProtocol: + type: "NATS" + nats: + url: "nats://example-nats.argo-events:4222" + clusterId: "example-stan" + clientId: "myclient" + type: "Streaming" + eventVersion: "1.0" + deploySpec: + metadata: + name: "webhook-gateway-nats-streaming" + labels: + gateway-name: "webhook-gateway-nats-streaming" + spec: + containers: + - name: "gateway-client" + image: "argoproj/gateway-client" + imagePullPolicy: "Always" + command: ["/bin/gateway-client"] + - name: "webhook-events" + image: "argoproj/webhook-gateway" + imagePullPolicy: "Always" + command: ["/bin/webhook-gateway"] + serviceAccountName: "argo-events-sa" + serviceSpec: + metadata: + name: webhook-gateway-svc + spec: + selector: + gateway-name: "webhook-gateway-nats-streaming" + ports: + - port: 12000 + targetPort: 12000 + type: LoadBalancer diff --git a/examples/gateways/webhook.yaml b/examples/gateways/webhook.yaml deleted file mode 100644 index 921887aced..0000000000 --- a/examples/gateways/webhook.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Gateway -metadata: - name: webhook-gateway - labels: - gateways.argoproj.io/gateway-controller-instanceid: argo-events - gateway-name: "webhook-gateway" -spec: - configMap: "webhook-gateway-configmap" - type: "webhook" - dispatchMechanism: "HTTP" - eventVersion: "1.0" - imageVersion: "latest" - deploySpec: - containers: - - name: "webhook-events" - image: "argoproj/webhook-gateway" - imagePullPolicy: "IfNotPresent" - command: ["/bin/webhook-gateway"] - serviceAccountName: "argo-events-sa" - transformerImage: "argoproj/gateway-http-transformer" - serviceSpec: - selector: - gateway-name: "webhook-gateway" - ports: - - port: 12000 - targetPort: 12000 - type: LoadBalancer - watchers: - sensors: - - name: "webhook-sensor" diff --git a/examples/sensors/amqp.yaml b/examples/sensors/amqp.yaml index 8867c18265..92879885d9 100644 --- a/examples/sensors/amqp.yaml +++ b/examples/sensors/amqp.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: amqp-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "amqp-gateway:foo" triggers: - name: amqp-workflow-trigger resource: diff --git a/examples/sensors/artifact-with-param-nats-standard.yaml b/examples/sensors/artifact-with-param-nats-standard.yaml new file mode 100644 index 0000000000..e130484c26 --- /dev/null +++ b/examples/sensors/artifact-with-param-nats-standard.yaml @@ -0,0 +1,47 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Sensor +metadata: + name: artifact-with-param-nats-standard-sensor + labels: + sensors.argoproj.io/sensor-controller-instanceid: argo-events +spec: + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "NATS" + nats: + type: "Standard" + url: "nats://example-nats.argo-events:4222" + dependencies: + - name: "artifact-gateway-nats-standard:input" + triggers: + - name: argo-workflow + resource: + namespace: argo-events + group: argoproj.io + version: v1alpha1 + kind: Workflow + # The container args from the workflow are overridden by the s3 notification key + parameters: + - src: + event: "artifact-gateway-nats-standard:input" + path: s3.object.key + dest: spec.templates.0.container.args.0 + source: + inline: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: artifact-workflow- + spec: + entrypoint: whalesay + templates: + - name: whalesay + container: + command: + - cowsay + image: "docker/whalesay:latest" diff --git a/examples/sensors/artifact-with-param-nats-streaming.yaml b/examples/sensors/artifact-with-param-nats-streaming.yaml new file mode 100644 index 0000000000..90621a7bbd --- /dev/null +++ b/examples/sensors/artifact-with-param-nats-streaming.yaml @@ -0,0 +1,49 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Sensor +metadata: + name: artifact-with-param-nats-streaming-sensor + labels: + sensors.argoproj.io/sensor-controller-instanceid: argo-events +spec: + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "NATS" + nats: + type: "Streaming" + url: "nats://example-nats.argo-events:4222" + clusterId: "example-stan" + clientId: "myclient1" + dependencies: + - name: "artifact-gateway-nats-streaming:input" + triggers: + - name: argo-workflow + resource: + namespace: argo-events + group: argoproj.io + version: v1alpha1 + kind: Workflow + # The container args from the workflow are overridden by the s3 notification key + parameters: + - src: + event: "artifact-gateway-nats-streaming:input" + path: s3.object.key + dest: spec.templates.0.container.args.0 + source: + inline: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: artifact-workflow- + spec: + entrypoint: whalesay + templates: + - name: whalesay + container: + command: + - cowsay + image: "docker/whalesay:latest" diff --git a/examples/sensors/s3-with-param.yaml b/examples/sensors/artifact-with-param.yaml similarity index 75% rename from examples/sensors/s3-with-param.yaml rename to examples/sensors/artifact-with-param.yaml index fd853c1a70..526ba55f43 100644 --- a/examples/sensors/s3-with-param.yaml +++ b/examples/sensors/artifact-with-param.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: artifact-gateway/input + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "artifact-gateway:input" triggers: - name: argo-workflow resource: @@ -21,7 +27,7 @@ spec: # The container args from the workflow are overridden by the s3 notification key parameters: - src: - signal: artifact-gateway/input + event: "artifact-gateway:input" path: s3.object.key dest: spec.templates.0.container.args.0 source: diff --git a/examples/sensors/s3.yaml b/examples/sensors/artifact.yaml similarity index 75% rename from examples/sensors/s3.yaml rename to examples/sensors/artifact.yaml index e5ebb256b7..19a63c20f4 100644 --- a/examples/sensors/s3.yaml +++ b/examples/sensors/artifact.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: artifact-gateway/input + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "artifact-gateway:input" triggers: - name: artifact-workflow-trigger resource: diff --git a/examples/sensors/calendar.yaml b/examples/sensors/calendar.yaml index 1453651264..025a918b31 100644 --- a/examples/sensors/calendar.yaml +++ b/examples/sensors/calendar.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: calendar-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + dependencies: + - name: "calendar-gateway:interval" + eventProtocol: + type: "HTTP" + http: + port: "9300" triggers: - name: calendar-workflow-trigger resource: diff --git a/examples/sensors/context-filter-webhook.yaml b/examples/sensors/context-filter-webhook.yaml index 72d6c0d013..60d107298f 100644 --- a/examples/sensors/context-filter-webhook.yaml +++ b/examples/sensors/context-filter-webhook.yaml @@ -5,17 +5,23 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: webhook-gateway/index + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + dependencies: + - name: "webhook-gateway:foo" filters: context: - source: - host: xyz.com - contentType: application/json + source: + host: xyz.com + contentType: application/json + eventProtocol: + type: "HTTP" + http: + port: "9300" triggers: - name: done-workflow resource: diff --git a/examples/sensors/data-filter-webhook.yaml b/examples/sensors/data-filter-webhook.yaml index cf1024d538..613e0b1abf 100644 --- a/examples/sensors/data-filter-webhook.yaml +++ b/examples/sensors/data-filter-webhook.yaml @@ -6,17 +6,23 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: webhook-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + dependencies: + - name: "webhook-gateway:foo" filters: data: - - path: bucket - type: string - value: argo-workflow-input + - path: bucket + type: string + value: argo-workflow-input + eventProtocol: + type: "HTTP" + http: + port: "9300" triggers: - name: done-workflow resource: diff --git a/examples/sensors/file-sensor.yaml b/examples/sensors/file-sensor.yaml index 3743d1b907..a98ca8eab0 100644 --- a/examples/sensors/file-sensor.yaml +++ b/examples/sensors/file-sensor.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: calendar-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + dependencies: + - name: "calendar-gateway:foo" + eventProtocol: + type: "HTTP" + http: + port: "9300" triggers: - name: file-workflow-trigger resource: diff --git a/examples/sensors/file.yaml b/examples/sensors/file.yaml index 4ef5452dee..8ec8fe6c4c 100644 --- a/examples/sensors/file.yaml +++ b/examples/sensors/file.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: file-gateway/bindir + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor:v0.6.3" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + dependencies: + - name: "file-gateway:bindir" + eventProtocol: + type: "HTTP" + http: + port: "9300" triggers: - name: file-workflow-trigger resource: diff --git a/examples/sensors/github.yaml b/examples/sensors/github.yaml index 53c47b39e2..5cdbe42153 100644 --- a/examples/sensors/github.yaml +++ b/examples/sensors/github.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: webhook-gateway/index + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + dependencies: + - name: "webhook-gateway-http:push" + eventProtocol: + type: "HTTP" + http: + port: "9300" triggers: - name: github-workflow-trigger resource: @@ -18,19 +24,31 @@ spec: group: argoproj.io version: v1alpha1 kind: Workflow + # The parameters from the workflow are overridden by the webhook's message + # An empty path means complete payload will be passed to trigger + parameters: + - src: + event: "webhook-gateway-http:push" + dest: spec.arguments.parameters.0.value source: inline: | apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - generateName: hello-world- + generateName: arguments-via-webhook-event spec: entrypoint: whalesay + arguments: + parameters: + - name: message + # this is the value that should be overridden + value: hello world templates: - - name: whalesay - container: - args: - - "hello world" - command: - - cowsay - image: "docker/whalesay:latest" + - name: whalesay + inputs: + parameters: + - name: message + container: + image: docker/whalesay:latest + command: [cowsay] + args: ["{{inputs.parameters.message}}"] diff --git a/examples/sensors/gitlab.yaml b/examples/sensors/gitlab.yaml index e873a0b1c5..60ce300527 100644 --- a/examples/sensors/gitlab.yaml +++ b/examples/sensors/gitlab.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: webhook-gateway/index + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "webhook-gateway-http:foo" triggers: - name: gitlab-workflow-trigger resource: diff --git a/examples/sensors/inline-sensor.yaml b/examples/sensors/inline-sensor.yaml index 6ac37cba03..0eb0b86f9b 100644 --- a/examples/sensors/inline-sensor.yaml +++ b/examples/sensors/inline-sensor.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: calendar-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "calendar-gateway:interval" triggers: - name: inline-workflow-trigger resource: diff --git a/examples/sensors/kafka.yaml b/examples/sensors/kafka.yaml index 444d5f804c..a60bb81f80 100644 --- a/examples/sensors/kafka.yaml +++ b/examples/sensors/kafka.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: kafka-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "kafka-gateway/foo" triggers: - name: kafka-workflow-trigger resource: diff --git a/examples/sensors/mqtt-sensor.yaml b/examples/sensors/mqtt-sensor.yaml index 4930b2e225..e2dc5779b8 100644 --- a/examples/sensors/mqtt-sensor.yaml +++ b/examples/sensors/mqtt-sensor.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: mqtt-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "mqtt-gateway:foo" triggers: - name: mqtt-workflow-trigger resource: diff --git a/examples/sensors/multi-signal-sensor.yaml b/examples/sensors/multi-signal-sensor.yaml index b647fcbca0..5e2926efd8 100644 --- a/examples/sensors/multi-signal-sensor.yaml +++ b/examples/sensors/multi-signal-sensor.yaml @@ -5,13 +5,19 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: webhook-gateway/foo - - name: calendar-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "webhook-gateway-http:foo" + - name: "calendar-gateway:interval" triggers: - name: multi-signal-workflow-trigger resource: diff --git a/examples/sensors/multi-trigger-sensor.yaml b/examples/sensors/multi-trigger-sensor.yaml index fbaa056c9d..bb91ad0679 100644 --- a/examples/sensors/multi-trigger-sensor.yaml +++ b/examples/sensors/multi-trigger-sensor.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: nats-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "webhook-gateway-http:foo" triggers: - name: multi-trigger-workflow-1 resource: @@ -20,8 +26,9 @@ spec: kind: Workflow source: s3: - bucket: workflows - key: hello-world.yaml + bucket: + name: workflows + key: hello-world.yaml endpoint: minio-service.argo-events:9000 insecure: true accessKey: diff --git a/examples/sensors/nats.yaml b/examples/sensors/nats.yaml index 09818ba570..752f7d2583 100644 --- a/examples/sensors/nats.yaml +++ b/examples/sensors/nats.yaml @@ -7,12 +7,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: nats-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "nats-gateway:foo" triggers: - name: nats-workflow-trigger resource: diff --git a/examples/sensors/resource.yaml b/examples/sensors/resource.yaml index 2e5c1c70cd..d19fdac33d 100644 --- a/examples/sensors/resource.yaml +++ b/examples/sensors/resource.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: resource-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "resource-gateway:foo" triggers: - name: argo-workflow resource: diff --git a/examples/sensors/storage-grid-sensor.yaml b/examples/sensors/storage-grid-sensor.yaml index c9fbcbc2ca..5471792870 100644 --- a/examples/sensors/storage-grid-sensor.yaml +++ b/examples/sensors/storage-grid-sensor.yaml @@ -1,16 +1,22 @@ apiVersion: argoproj.io/v1alpha1 kind: Sensor metadata: - name: storage_grid-watcher-sensor + name: storage-grid-watcher-sensor labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: storage-grid-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + dependencies: + - name: "storage-grid-gateway:my_bucket" + eventProtocol: + type: "HTTP" + http: + port: "9300" triggers: - name: argo-workflow resource: @@ -20,7 +26,7 @@ spec: kind: Workflow parameters: - src: - signal: storage-grid-gateway/foo + event: "storage-grid-gateway:my_bucket" path: value: hello world dest: spec.arguments.parameters.0.value @@ -29,8 +35,9 @@ spec: apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - name: arguments-via-webhook-event + generateName: storage-grid-wf- spec: + serviceAccountName: argo-events-sa entrypoint: whalesay arguments: parameters: diff --git a/examples/sensors/time-filter-webhook.yaml b/examples/sensors/time-filter-webhook.yaml index 433fd6469f..3b44450efe 100644 --- a/examples/sensors/time-filter-webhook.yaml +++ b/examples/sensors/time-filter-webhook.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: webhook-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "webhook-gateway-http:foo" filters: time: start: "22:14:05" diff --git a/examples/sensors/url-sensor.yaml b/examples/sensors/url-sensor.yaml index 77cbcdad78..669d2135bb 100644 --- a/examples/sensors/url-sensor.yaml +++ b/examples/sensors/url-sensor.yaml @@ -5,12 +5,18 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: calendar-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" + dependencies: + - name: "artifact-gateway:input" triggers: - name: url-workflow-trigger resource: diff --git a/examples/sensors/webhook.yaml b/examples/sensors/webhook-http.yaml similarity index 72% rename from examples/sensors/webhook.yaml rename to examples/sensors/webhook-http.yaml index 4de14a3e83..78db19e201 100644 --- a/examples/sensors/webhook.yaml +++ b/examples/sensors/webhook-http.yaml @@ -1,17 +1,22 @@ - apiVersion: argoproj.io/v1alpha1 kind: Sensor metadata: - name: webhook-sensor + name: webhook-sensor-http labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: webhook-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + dependencies: + - name: "webhook-gateway-http:foo" + eventProtocol: + type: "HTTP" + http: + port: "9300" triggers: - name: webhook-workflow-trigger resource: diff --git a/examples/sensors/webhook-nats-streaming.yaml b/examples/sensors/webhook-nats-streaming.yaml new file mode 100644 index 0000000000..464294085d --- /dev/null +++ b/examples/sensors/webhook-nats-streaming.yaml @@ -0,0 +1,58 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Sensor +metadata: + name: webhook-nats-streaming + labels: + sensors.argoproj.io/sensor-controller-instanceid: argo-events +spec: + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "NATS" + nats: + type: "Streaming" + url: "nats://example-nats.argo-events:4222" + clusterId: "example-stan" + clientId: "myclient1" + deliverAllAvailable: true + dependencies: + - name: "webhook-gateway-nats-streaming:foo" + triggers: + - name: argo-workflow + resource: + namespace: argo-events + group: argoproj.io + version: v1alpha1 + kind: Workflow + # The parameters from the workflow are overridden by the webhook's message + # An empty path means complete payload will be passed to trigger + parameters: + - src: + event: "webhook-gateway-nats-streaming:foo" + dest: spec.arguments.parameters.0.value + source: + inline: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: arguments-via-webhook-event + spec: + entrypoint: whalesay + arguments: + parameters: + - name: message + # this is the value that should be overridden + value: hello world + templates: + - name: whalesay + inputs: + parameters: + - name: message + container: + image: docker/whalesay:latest + command: [cowsay] + args: ["{{inputs.parameters.message}}"] diff --git a/examples/sensors/webhook-nats.yaml b/examples/sensors/webhook-nats.yaml new file mode 100644 index 0000000000..f68540708e --- /dev/null +++ b/examples/sensors/webhook-nats.yaml @@ -0,0 +1,43 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Sensor +metadata: + name: webhook-sensor + labels: + sensors.argoproj.io/sensor-controller-instanceid: argo-events +spec: + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor:v0.6.3" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + dependencies: + - name: "webhook-gateway-nats:foo" + eventProtocol: + type: "NATS" + nats: + url: "nats://nats.argo-events:4222" + type: "Standard" + triggers: + - name: webhook-workflow-trigger + resource: + namespace: argo-events + group: argoproj.io + version: v1alpha1 + kind: Workflow + source: + inline: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: hello-world- + spec: + entrypoint: whalesay + templates: + - name: whalesay + container: + args: + - "hello world" + command: + - cowsay + image: "docker/whalesay:latest" diff --git a/examples/sensors/webhook-with-complete-payload.yaml b/examples/sensors/webhook-with-complete-payload.yaml index fc6bf1bf88..36dc62691b 100644 --- a/examples/sensors/webhook-with-complete-payload.yaml +++ b/examples/sensors/webhook-with-complete-payload.yaml @@ -5,12 +5,19 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: webhook-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor:v0.6.2" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "NATS" + nats: + type: "Standard" + url: "nats://nats.argo-events:4222" + dependencies: + - name: "webhook-gateway-nats:foo" triggers: - name: argo-workflow resource: @@ -22,14 +29,14 @@ spec: # An empty path means complete payload will be passed to trigger parameters: - src: - signal: webhook-gateway/foo + event: "webhook-gateway-nats:foo" dest: spec.arguments.parameters.0.value source: inline: | apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: - name: arguments-via-webhook-event + generateName: arguments-via-webhook-event spec: entrypoint: whalesay arguments: diff --git a/examples/sensors/webhook-with-resource-param.yaml b/examples/sensors/webhook-with-resource-param.yaml index 3594d337ad..bc8ad8b8aa 100644 --- a/examples/sensors/webhook-with-resource-param.yaml +++ b/examples/sensors/webhook-with-resource-param.yaml @@ -5,12 +5,19 @@ metadata: labels: sensors.argoproj.io/sensor-controller-instanceid: argo-events spec: - repeat: true - serviceAccountName: argo-events-sa - imageVersion: "latest" - imagePullPolicy: Always - signals: - - name: webhook-gateway/foo + deploySpec: + containers: + - name: "sensor" + image: "argoproj/sensor:v0.6.2" + imagePullPolicy: Always + serviceAccountName: argo-events-sa + eventProtocol: + type: "NATS" + nats: + type: "Standard" + url: "nats://nats.argo-events:4222" + dependencies: + - name: "webhook-gateway-nats:foo" triggers: - name: argo-workflow resource: @@ -19,31 +26,32 @@ spec: version: v1alpha1 kind: Workflow # The parameters from the workflow are overridden by the webhook's message + # An empty path means complete payload will be passed to trigger parameters: - src: - signal: webhook-gateway/foo - path: message - value: hello default + event: "webhook-gateway-nats:foo" + path: "index.msg" + value: "default value" dest: spec.arguments.parameters.0.value source: inline: | - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - name: arguments-via-webhook-event - spec: - entrypoint: whalesay - arguments: + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: arguments-via-webhook-event + spec: + entrypoint: whalesay + arguments: + parameters: + - name: message + # this is the value that should be overridden + value: hello world + templates: + - name: whalesay + inputs: parameters: - name: message - # this is the value that should be overridden - value: hello world - templates: - - name: whalesay - inputs: - parameters: - - name: message - container: - image: docker/whalesay:latest - command: [cowsay] - args: ["{{inputs.parameters.message}}"] + container: + image: docker/whalesay:latest + command: [cowsay] + args: ["{{inputs.parameters.message}}"] diff --git a/gateways/custom/github/Dockerfile b/gateways/community/github/Dockerfile similarity index 100% rename from gateways/custom/github/Dockerfile rename to gateways/community/github/Dockerfile diff --git a/gateways/custom/github/cmd/main.go b/gateways/community/github/cmd/main.go similarity index 95% rename from gateways/custom/github/cmd/main.go rename to gateways/community/github/cmd/main.go index e85b37d345..43d0d486f8 100644 --- a/gateways/custom/github/cmd/main.go +++ b/gateways/community/github/cmd/main.go @@ -19,7 +19,7 @@ package main import ( "github.com/argoproj/argo-events/common" "github.com/argoproj/argo-events/gateways" - "github.com/argoproj/argo-events/gateways/custom/github" + "github.com/argoproj/argo-events/gateways/community/github" "k8s.io/client-go/kubernetes" "os" ) diff --git a/gateways/custom/github/config.go b/gateways/community/github/config.go similarity index 100% rename from gateways/custom/github/config.go rename to gateways/community/github/config.go diff --git a/gateways/custom/github/start.go b/gateways/community/github/start.go similarity index 100% rename from gateways/custom/github/start.go rename to gateways/community/github/start.go diff --git a/gateways/custom/github/tokenauth.go b/gateways/community/github/tokenauth.go similarity index 100% rename from gateways/custom/github/tokenauth.go rename to gateways/community/github/tokenauth.go diff --git a/gateways/custom/github/validate.go b/gateways/community/github/validate.go similarity index 100% rename from gateways/custom/github/validate.go rename to gateways/community/github/validate.go diff --git a/gateways/custom/github/validate_test.go b/gateways/community/github/validate_test.go similarity index 100% rename from gateways/custom/github/validate_test.go rename to gateways/community/github/validate_test.go diff --git a/gateways/custom/gitlab/Dockerfile b/gateways/community/gitlab/Dockerfile similarity index 100% rename from gateways/custom/gitlab/Dockerfile rename to gateways/community/gitlab/Dockerfile diff --git a/gateways/custom/gitlab/cmd/main.go b/gateways/community/gitlab/cmd/main.go similarity index 95% rename from gateways/custom/gitlab/cmd/main.go rename to gateways/community/gitlab/cmd/main.go index 956bb7e2ea..5a3653e7e5 100644 --- a/gateways/custom/gitlab/cmd/main.go +++ b/gateways/community/gitlab/cmd/main.go @@ -21,7 +21,7 @@ import ( "github.com/argoproj/argo-events/common" "github.com/argoproj/argo-events/gateways" - "github.com/argoproj/argo-events/gateways/custom/gitlab" + "github.com/argoproj/argo-events/gateways/community/gitlab" "k8s.io/client-go/kubernetes" ) diff --git a/gateways/custom/gitlab/config.go b/gateways/community/gitlab/config.go similarity index 100% rename from gateways/custom/gitlab/config.go rename to gateways/community/gitlab/config.go diff --git a/gateways/custom/gitlab/start.go b/gateways/community/gitlab/start.go similarity index 100% rename from gateways/custom/gitlab/start.go rename to gateways/community/gitlab/start.go diff --git a/gateways/custom/gitlab/validate.go b/gateways/community/gitlab/validate.go similarity index 100% rename from gateways/custom/gitlab/validate.go rename to gateways/community/gitlab/validate.go diff --git a/gateways/custom/gitlab/validate_test.go b/gateways/community/gitlab/validate_test.go similarity index 100% rename from gateways/custom/gitlab/validate_test.go rename to gateways/community/gitlab/validate_test.go diff --git a/gateways/custom/storagegrid/Dockerfile b/gateways/community/storagegrid/Dockerfile similarity index 100% rename from gateways/custom/storagegrid/Dockerfile rename to gateways/community/storagegrid/Dockerfile diff --git a/gateways/custom/storagegrid/cmd/main.go b/gateways/community/storagegrid/cmd/main.go similarity index 92% rename from gateways/custom/storagegrid/cmd/main.go rename to gateways/community/storagegrid/cmd/main.go index 76c9f7f5a8..e103870c60 100644 --- a/gateways/custom/storagegrid/cmd/main.go +++ b/gateways/community/storagegrid/cmd/main.go @@ -19,7 +19,7 @@ package main import ( "github.com/argoproj/argo-events/common" "github.com/argoproj/argo-events/gateways" - "github.com/argoproj/argo-events/gateways/custom/storagegrid" + "github.com/argoproj/argo-events/gateways/community/storagegrid" ) func main() { diff --git a/gateways/custom/storagegrid/config.go b/gateways/community/storagegrid/config.go similarity index 100% rename from gateways/custom/storagegrid/config.go rename to gateways/community/storagegrid/config.go diff --git a/gateways/custom/storagegrid/start.go b/gateways/community/storagegrid/start.go similarity index 100% rename from gateways/custom/storagegrid/start.go rename to gateways/community/storagegrid/start.go diff --git a/gateways/custom/storagegrid/validate.go b/gateways/community/storagegrid/validate.go similarity index 100% rename from gateways/custom/storagegrid/validate.go rename to gateways/community/storagegrid/validate.go diff --git a/gateways/custom/storagegrid/validate_test.go b/gateways/community/storagegrid/validate_test.go similarity index 100% rename from gateways/custom/storagegrid/validate_test.go rename to gateways/community/storagegrid/validate_test.go diff --git a/gateways/core/artifact/install.md b/gateways/core/artifact/install.md new file mode 100644 index 0000000000..aeb49b10e2 --- /dev/null +++ b/gateways/core/artifact/install.md @@ -0,0 +1,60 @@ +# Artifact Gateway & Sensor + +Minio bucket notifications acts as event sources for artifact gateway. To setup minio, follow https://www.minio.io/kubernetes.html + +1. [Example event sources definition](#example-event-sources-definition) +2. [Install gateway](#install-gateway) +3. [Install sensor](#install-sensor) +4. [Trigger Workflow](#trigger-workflow) + +## Example event sources definition +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: artifact-gateway-configmap +data: + input: |- + bucket: + name: input # name of the bucket we want to listen to + endpoint: minio-service.argo-events:9000 # minio service endpoint + event: s3:ObjectCreated:Put # type of event + filter: # filter on object name if any + prefix: "" + suffix: "" + insecure: true # type of minio server deployment + accessKey: + key: accesskey # key within below k8 secret whose corresponding value is name of the accessKey + name: artifacts-minio # k8 secret name that holds minio creds + secretKey: + key: secretkey # key within below k8 secret whose corresponding value is name of the secretKey + name: artifacts-minio # k8 secret name that holds minio creds +``` + +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/artifact-gateway-configmap.yaml +``` + +## Install Gateway +Pre-requisite - create necessary buckets in Minio. +1. **Create gateway** + + ```yaml + kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/artifact-http.yaml + ``` + +2. **Check the status of the gateway** + + ```yaml + kubectl -n argo-events describe gateway artifact-gateway + ``` + + Make sure the gateway is in active state and all the event sources are in running state. + +## Install Sensor +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/sensors/artifact.yaml +``` + +## Trigger workflow +Drop a file onto `input` bucket and monitor workflows \ No newline at end of file diff --git a/gateways/core/calendar/install.md b/gateways/core/calendar/install.md new file mode 100644 index 0000000000..1593209e4b --- /dev/null +++ b/gateways/core/calendar/install.md @@ -0,0 +1,49 @@ +# Calendar Gateway & Sensor + +Intervals or cron schedules act as event sources for calendar gateway. + +1. [Example event sources definition](#example-event-sources-definition) +2. [Install gateway](#install-gateway) +3. [Install sensor](#install-sensor) +4. [Trigger Workflow](#trigger-workflow) + +## Example event sources definition +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: calendar-gateway-configmap +data: + interval: |- + interval: 55s # event is generated after every 10 seconds + schedule: |- + schedule: 30 * * * * # event is generated after 30 min past every hour +``` + +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/calendar-gateway-configmap.yaml +``` + +## Install gateway +1. **Create gateway** + + ```yaml + kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/calendar.yaml + ``` + +2. **Check the status of the gateway** + + ```yaml + kubectl -n argo-events describe gateway calendar-gateway + ``` + + Make sure the gateway is in active state and all the event sources are in running state. + +## Install Sensor +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/sensors/calendar.yaml +``` + +## Trigger Workflow +Wait for 55 seconds to pass or if the sensor has `calendar-gateway:schedule` as event dependency, then wait for 30 min past each hour +for workflow to trigger. \ No newline at end of file diff --git a/gateways/core/file/install.md b/gateways/core/file/install.md new file mode 100644 index 0000000000..ae5bb43d8b --- /dev/null +++ b/gateways/core/file/install.md @@ -0,0 +1,57 @@ +# File Gateway & Sensor + +File system serves as event source for file gateway + +1. [Example event sources definition](#example-event-sources-definition) +2. [Install gateway](#install-gateway) +3. [Install sensor](#install-sensor) +4. [Trigger Workflow](#trigger-workflow) + +## Example event sources definition +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: file-gateway-configmap +data: + bindir: |- # event source name can be any valid string + directory: "/bin/" # directory where file events are watched + type: CREATE # type of file event + path: x.txt # file to watch to +``` + +Supported file events are - +* Create +* Write +* Remove +* Rename +* Chmod + +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/file-gateway-configmap.yaml +``` + +## Install gateway +Pre-requisite - The file system you want to watch must be mounted in gateway pod and the directory under which a file is to be watched must exist. + +1. **Create gateway** + + ```yaml + kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/file.yaml + ``` + +2. **Check the status of the gateway** + + ```yaml + kubectl -n argo-events describe gateway file-gateway + ``` + + Make sure the gateway is in active state and all the event sources are in running state. + +## Install Sensor +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/sensors/file.yaml +``` + +## Trigger Workflow +Go to the directory specified in event source and create a file. diff --git a/gateways/core/resource/config.go b/gateways/core/resource/config.go index 5077f8d409..76047545a4 100644 --- a/gateways/core/resource/config.go +++ b/gateways/core/resource/config.go @@ -31,7 +31,6 @@ type ResourceEventSourceExecutor struct { } // resource refers to a dependency on a k8s resource. -// +k8s:openapi-gen=true type resource struct { // Namespace where resource is deployed Namespace string `json:"namespace"` @@ -40,18 +39,15 @@ type resource struct { // Version of the source Version string `json:"version"` // Group of the resource - // +k8s:openapi-gen=false metav1.GroupVersionKind `json:",inline"` } // ResourceFilter contains K8 ObjectMeta information to further filter resource event objects -// +k8s:openapi-gen=true type ResourceFilter struct { Prefix string `json:"prefix,omitempty"` Labels map[string]string `json:"labels,omitempty"` Annotations map[string]string `json:"annotations,omitempty"` - // +k8s:openapi-gen=false - CreatedBy metav1.Time `json:"createdBy,omitempty"` + CreatedBy metav1.Time `json:"createdBy,omitempty"` } func parseEventSource(es string) (*resource, error) { diff --git a/gateways/core/resource/install.md b/gateways/core/resource/install.md new file mode 100644 index 0000000000..9e2419432c --- /dev/null +++ b/gateways/core/resource/install.md @@ -0,0 +1,64 @@ +# Resource Gateway & Sensor + +K8s resources act as event sources for Resource gateway + +1. [Example event sources definition](#example-event-sources-definition) +2. [Install gateway](#install-gateway) +3. [Install sensor](#install-sensor) +4. [Trigger Workflow](#trigger-workflow) + +## Example event sources definition +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: resource-gateway-configmap +data: + successWorkflow: |- + namespace: argo-events # namespace where wf is deloyef + group: "argoproj.io" # wf group + version: "v1alpha1" # wf version + kind: "Workflow" # object kind + filter: # filters can be applied on labels, annotations, creation time and name + labels: + workflows.argoproj.io/phase: Succeeded + name: "my-workflow" + failureWorkflow: |- + namespace: argo-events + group: "argoproj.io" + version: "v1alpha1" + kind: "Workflow" + filter: + prefix: scripts-bash + labels: + workflows.argoproj.io/phase: Failed +``` + +Create gateway event sources + +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/resource-gateway-configmap.yaml +``` + +## Install gateway +1. **Create gateway** + + ```yaml + kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/resource.yaml + ``` + +2. **Check the status of the gateway** + + ```yaml + kubectl -n argo-events describe gateway resource-gateway + ``` + + Make sure the gateway is in active state and all the event sources are in running state. + +## Install Sensor +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/sensors/resource.yaml +``` + +## Trigger workflow +Create an argo workflow with name `my-workflow`. As soon as `my-workflow` succeeds, a new workflow will be triggered. \ No newline at end of file diff --git a/gateways/core/stream/amqp/install.md b/gateways/core/stream/amqp/install.md new file mode 100644 index 0000000000..1ef4d555eb --- /dev/null +++ b/gateways/core/stream/amqp/install.md @@ -0,0 +1,54 @@ +# AMQP Gateway & Sensor + +1. [Example event sources definition](#example-event-sources-definition) +2. [Install gateway](#install-gateway) +3. [Install sensor](#install-sensor) +4. [Trigger Workflow](#trigger-workflow) + +## Example event sources definition +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: amqp-gateway-configmap +data: + foo: |- + url: amqp://amqp.argo-events:5672 + exchangeName: foo + exchangeType: fanout + routingKey: fooK + bar: |- + url: amqp://amqp.argo-events:5672 + exchangeName: bar + exchangeType: fanout + routingKey: barK +``` + +Create gateway event sources + +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/amqp-gateway-configmap.yaml +``` + +## Install gateway +1. **Create gateway** + + ```yaml + kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/amqp.yaml + ``` + +2. **Check the status of the gateway** + + ```yaml + kubectl -n argo-events describe gateway amqp-gateway + ``` + + Make sure the gateway is in active state and all the event sources are in running state. + +## Install Sensor +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/sensors/amqp.yaml +``` + +## Trigger workflow +Send a message to exchange name `foo` with exchange type `fanout` and routing key as `fooK` on amqp://amqp.argo-events:5672 \ No newline at end of file diff --git a/gateways/core/stream/kafka/install.md b/gateways/core/stream/kafka/install.md new file mode 100644 index 0000000000..fcc10bdaff --- /dev/null +++ b/gateways/core/stream/kafka/install.md @@ -0,0 +1,54 @@ +# KAFKA Gateway & Sensor + +Kafka topics act as event sources for gateway + +1. [Example event sources definition](#example-event-sources-definition) +2. [Install gateway](#install-gateway) +3. [Install sensor](#install-sensor) +4. [Trigger Workflow](#trigger-workflow) + +## Example event sources definition +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: kafka-gateway-configmap +data: + foo: |- + url: kafka.argo-events:9092 # kafka service + topic: foo # topic name + partition: "0" # topic partition + bar: |- + url: kafka.argo-events:9092 + topic: bar + partition: "1" +``` + +Create gateway event sources + +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/kafka-gateway-configmap.yaml +``` + +## Install gateway +1. **Create gateway** + + ```yaml + kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/kafka.yaml + ``` + +2. **Check the status of the gateway** + + ```yaml + kubectl -n argo-events describe gateway kafka-gateway + ``` + + Make sure the gateway is in active state and all the event sources are in running state. + +## Install Sensor +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/sensors/kafka.yaml +``` + +## Trigger Workflow +Send a message to topic `foo` on partition `0`. You might find this useful https://kafka.apache.org/quickstart#quickstart_send \ No newline at end of file diff --git a/gateways/core/stream/mqtt/install.md b/gateways/core/stream/mqtt/install.md new file mode 100644 index 0000000000..8f2c50db47 --- /dev/null +++ b/gateways/core/stream/mqtt/install.md @@ -0,0 +1,50 @@ +# MQTT Gateway & Sensor + +1. [Example event sources definition](#example-event-sources-definition) +2. [Install gateway](#install-gateway) +3. [Install sensor](#install-sensor) +4. [Trigger Workflow](#trigger-workflow) + +## Example event sources definition +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: mqtt-gateway-configmap +data: + foo: |- + url: tcp://mqtt.argo-events:1883 # mqtt service + topic: foo # topic to listen to + bar: |- + url: tcp://mqtt.argo-events:1883 + topic: bar +``` + +Create gateway event sources + +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/mqtt-gateway-configmap.yaml +``` + +## Install gateway +1. **Create gateway** + + ```yaml + kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/mqtt.yaml + ``` + +2. **Check the status of the gateway** + + ```yaml + kubectl -n argo-events describe gateway mqtt-gateway + ``` + + Make sure the gateway is in active state and all the event sources are in running state. + +## Install Sensor +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/sensors/mqtt.yaml +``` + +## Trigger Workflow +Publish message to topic `foo`. You might find this helpful https://www.ev3dev.org/docs/tutorials/sending-and-receiving-messages-with-mqtt/ \ No newline at end of file diff --git a/gateways/core/stream/nats/install.md b/gateways/core/stream/nats/install.md new file mode 100644 index 0000000000..43d6849e4f --- /dev/null +++ b/gateways/core/stream/nats/install.md @@ -0,0 +1,49 @@ +# NATS Gateway & Sensor + +NATS subjects act as event sources for gateway + +1. [Example event sources definition](#example-event-sources-definition) +2. [Install gateway](#install-gateway) +3. [Install sensor](#install-sensor) +4. [Trigger Workflow](#trigger-workflow) + +## Example event sources definition +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: nats-gateway-configmap +data: + foo: |- + url: nats://nats.argo-events:4222 # nats service + subject: foo # subject to listen to +``` + +Create gateway event sources + +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/nats-gateway-configmap.yaml +``` + +## Install gateway +1. **Create gateway** + + ```yaml + kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/nats.yaml + ``` + +2. **Check the status of the gateway** + + ```yaml + kubectl -n argo-events describe gateway nats-gateway + ``` + + Make sure the gateway is in active state and all the event sources are in running state. + +## Install Sensor +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/sensors/nats.yaml +``` + +## Trigger Workflow +Publish message to subject `foo`. You might find this useful https://github.com/nats-io/go-nats/tree/master/examples/nats-pub diff --git a/gateways/core/webhook/install.md b/gateways/core/webhook/install.md new file mode 100644 index 0000000000..934292d02f --- /dev/null +++ b/gateways/core/webhook/install.md @@ -0,0 +1,68 @@ +# Webhook Gateway & Sensor + +REST api endpoints act as event sources for gateway + +1. [Example event sources definition](#example-event-sources-definition) +2. [Install gateway](#install-gateway) +3. [Install sensor](#install-sensor) +4. [Trigger Workflow](#trigger-workflow) + +## Example event sources definition +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: webhook-gateway-configmap +data: + # each event source defines - + # the port for HTTP server + # endpoint to listen to + # acceptable http method + bar: |- + port: "12000" + endpoint: "/bar" + method: "POST" + foo: |- + port: "12000" + endpoint: "/foo" + method: "POST" +``` + +Create gateway event sources + +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/webhook-gateway-configmap.yaml +``` + +## Install gateway +1. **Create gateway** + + ```yaml + kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/gateways/webhook-http.yaml + ``` + +2. **Check the status of the gateway** + + ```yaml + kubectl -n argo-events describe gateway webhook-gateway-http + ``` + + Make sure the gateway is in active state and all the event sources are in running state. + +## Install Sensor +```yaml +kubectl -n argo-events create -f https://github.com/argoproj/argo-events/blob/master/examples/sensors/webhook-http.yaml +``` + +## Trigger Workflow +Note: the `WEBHOOK_SERVICE_URL` will differ based on the Kubernetes cluster. +``` +export WEBHOOK_SERVICE_URL=$(minikube service -n argo-events --url ) +echo $WEBHOOK_SERVICE_URL +curl -d '{"message":"this is my first webhook"}' -H "Content-Type: application/json" -X POST $WEBHOOK_SERVICE_URL/foo +``` + +Note: + * If you are facing an issue getting service url by running `minikube service -n argo-events --url `, you can use `kubectl port-forward` + * Open another terminal window and enter `kubectl port-forward -n argo-events 9003:` + * You can now use `localhost:9003` to query webhook gateway diff --git a/hack/k8s/manifests/argo-events-cluster-roles.yaml b/hack/k8s/manifests/argo-events-cluster-roles.yaml index c12a754af2..be11347ab8 100644 --- a/hack/k8s/manifests/argo-events-cluster-roles.yaml +++ b/hack/k8s/manifests/argo-events-cluster-roles.yaml @@ -5,7 +5,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: argo-events-cluster-role + name: argo-events-role subjects: - kind: ServiceAccount name: argo-events-sa @@ -14,7 +14,7 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: argo-events-cluster-role + name: argo-events-role rules: - apiGroups: - apiextensions.k8s.io diff --git a/hack/k8s/manifests/gateway-controller-deployment.yaml b/hack/k8s/manifests/gateway-controller-deployment.yaml index a67e4c4ee3..8e269085f5 100644 --- a/hack/k8s/manifests/gateway-controller-deployment.yaml +++ b/hack/k8s/manifests/gateway-controller-deployment.yaml @@ -13,7 +13,7 @@ spec: serviceAccountName: argo-events-sa containers: - name: gateway-controller - image: argoproj/gateway-controller:v0.6 + image: argoproj/gateway-controller imagePullPolicy: Always env: - name: GATEWAY_NAMESPACE diff --git a/hack/k8s/manifests/sensor-controller-configmap.yaml b/hack/k8s/manifests/sensor-controller-configmap.yaml index b4da0b62c8..63a2459cd6 100644 --- a/hack/k8s/manifests/sensor-controller-configmap.yaml +++ b/hack/k8s/manifests/sensor-controller-configmap.yaml @@ -8,4 +8,4 @@ metadata: data: config: | instanceID: argo-events - namespace: argo-events + namespace: argo-events \ No newline at end of file diff --git a/hack/k8s/manifests/sensor-controller-deployment.yaml b/hack/k8s/manifests/sensor-controller-deployment.yaml index f851101b57..343eec737d 100644 --- a/hack/k8s/manifests/sensor-controller-deployment.yaml +++ b/hack/k8s/manifests/sensor-controller-deployment.yaml @@ -13,7 +13,7 @@ spec: serviceAccountName: argo-events-sa containers: - name: sensor-controller - image: argoproj/sensor-controller:v0.6 + image: argoproj/sensor-controller imagePullPolicy: Always env: - name: SENSOR_NAMESPACE diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 5ad8540629..2c90f2204f 100644 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -21,6 +21,7 @@ set -o pipefail CODEGEN_PKG="../vendor/k8s.io/code-generator/" + bash -x ${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \ github.com/argoproj/argo-events/pkg/client/sensor github.com/argoproj/argo-events/pkg/apis \ "sensor:v1alpha1" \ @@ -30,3 +31,4 @@ bash -x ${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \ github.com/argoproj/argo-events/pkg/client/gateway github.com/argoproj/argo-events/pkg/apis \ "gateway:v1alpha1" \ --go-header-file ./custom-boilerplate.go.txt + diff --git a/pkg/apis/sensor/v1alpha1/build/generated.json b/pkg/apis/sensor/v1alpha1/build/generated.json deleted file mode 100644 index 76e4554e1a..0000000000 --- a/pkg/apis/sensor/v1alpha1/build/generated.json +++ /dev/null @@ -1,646 +0,0 @@ -{ - "swagger": "2.0", - "info": { - "title": "Argo-Events", - "version": "1.0" - }, - "paths": {}, - "definitions": { - "v1alpha1.ArtifactLocation": { - "description": "ArtifactLocation describes the source location for an external artifact", - "properties": { - "configmap": { - "$ref": "#/definitions/v1alpha1.ConfigmapArtifact" - }, - "file": { - "$ref": "#/definitions/v1alpha1.FileArtifact" - }, - "inline": { - "type": "string" - }, - "s3": { - "$ref": "#/definitions/v1alpha1.S3Artifact" - }, - "url": { - "$ref": "#/definitions/v1alpha1.URLArtifact" - } - } - }, - "v1alpha1.ConfigmapArtifact": { - "description": "ConfigmapArtifact contains information about artifact in k8 configmap", - "required": [ - "name", - "namespace", - "key" - ], - "properties": { - "key": { - "description": "Key within configmap data which contains trigger resource definition", - "type": "string" - }, - "name": { - "description": "Name of the configmap", - "type": "string" - }, - "namespace": { - "description": "Namespace where configmap is deployed", - "type": "string" - } - } - }, - "v1alpha1.Data": { - "required": [ - "filters" - ], - "properties": { - "escalationPolicy": { - "description": "EscalationPolicy is the escalation to trigger in case the signal filter fails", - "$ref": "#/definitions/v1alpha1.EscalationPolicy" - }, - "filters": { - "description": "filter constraints", - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1.DataFilter" - } - } - } - }, - "v1alpha1.DataFilter": { - "description": "DataFilter describes constraints and filters for event data Regular Expressions are purposefully not a feature as they are overkill for our uses here See Rob Pike's Post: https://commandcenter.blogspot.com/2011/08/regular-expressions-in-lexing-and.html", - "required": [ - "path", - "type", - "value" - ], - "properties": { - "escalationPolicy": { - "description": "EscalationPolicy is the escalation to trigger in case the signal filter fails", - "$ref": "#/definitions/v1alpha1.EscalationPolicy" - }, - "path": { - "description": "Path is the JSONPath of the event's (JSON decoded) data key Path is a series of keys separated by a dot. A key may contain wildcard characters '*' and '?'. To access an array value use the index as the key. The dot and wildcard characters can be escaped with '\\'. See https://github.com/tidwall/gjson#path-syntax for more information on how to use this.", - "type": "string" - }, - "type": { - "description": "Type contains the JSON type of the data", - "type": "string" - }, - "value": { - "description": "Value is the expected string value for this key Booleans are pased using strconv.ParseBool() Numbers are parsed using as float64 using strconv.ParseFloat() Strings are taken as is Nils this value is ignored", - "type": "string" - } - } - }, - "v1alpha1.EscalationPolicy": { - "description": "EscalationPolicy describes the policy for escalating sensors in an Error state. An escalation policy is associated with signal filter. Whenever a signal filter fails escalation will be triggered", - "required": [ - "name", - "level", - "message" - ], - "properties": { - "level": { - "description": "Level is the degree of importance", - "type": "string" - }, - "message": { - "description": "need someway to progressively get more serious notifications", - "type": "string" - }, - "name": { - "description": "Name is name of the escalation policy This is referred by signal filter/s", - "type": "string" - } - } - }, - "v1alpha1.Event": { - "description": "Event is a data record expressing an occurrence and its context. Adheres to the CloudEvents v0.1 specification", - "required": [ - "context", - "payload" - ], - "properties": { - "context": { - "$ref": "#/definitions/v1alpha1.EventContext" - }, - "payload": { - "type": "string", - "format": "byte" - } - } - }, - "v1alpha1.EventContext": { - "description": "EventContext contains metadata that provides circumstantial information about the occurrence.", - "required": [ - "eventType", - "eventTypeVersion", - "cloudEventsVersion", - "source", - "eventID", - "schemaURL", - "contentType" - ], - "properties": { - "cloudEventsVersion": { - "description": "The version of the CloudEvents specification which the event uses. Enables the interpretation of the context.", - "type": "string" - }, - "contentType": { - "description": "Content type of the data attribute value. Enables the data attribute to carry any type of content, whereby format and encoding might differ from that of the chosen event format. For example, the data attribute may carry an XML or JSON payload and the consumer is informed by this attribute being set to \"application/xml\" or \"application/json\" respectively.", - "type": "string" - }, - "escalationPolicy": { - "description": "EscalationPolicy is the name of escalation policy to trigger in case the signal filter fails", - "$ref": "#/definitions/v1alpha1.EscalationPolicy" - }, - "eventID": { - "description": "ID of the event. The semantics are explicitly undefined to ease the implementation of producers. Enables deduplication. Must be unique within scope of producer.", - "type": "string" - }, - "eventType": { - "description": "The type of occurrence which has happened. Often this attribute is used for routing, observability, policy enforcement, etc. should be prefixed with a reverse-DNS name. The prefixed domain dictates the organization which defines the semantics of this event type. ex: com.github.pull.create", - "type": "string" - }, - "eventTypeVersion": { - "description": "The version of the eventType. Enables the interpretation of data by eventual consumers, requires the consumer to be knowledgeable about the producer.", - "type": "string" - }, - "extensions": { - "description": "This is for additional metadata and does not have a mandated structure. Enables a place for custom fields a producer or middleware might want to include and provides a place to test metadata before adding them to the CloudEvents specification.", - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "schemaURL": { - "description": "A link to the schema that the data attribute adheres to. Must adhere to the format specified in RFC 3986.", - "$ref": "#/definitions/v1alpha1.URI" - }, - "source": { - "description": "This describes the event producer.", - "$ref": "#/definitions/v1alpha1.URI" - } - } - }, - "v1alpha1.EventWrapper": { - "description": "EventWrapper wraps an event with an additional flag to check if we processed this event already", - "required": [ - "event", - "seen" - ], - "properties": { - "event": { - "$ref": "#/definitions/v1alpha1.Event" - }, - "seen": { - "type": "boolean" - } - } - }, - "v1alpha1.FileArtifact": { - "description": "FileArtifact contains information about an artifact in a filesystem", - "properties": { - "path": { - "type": "string" - } - } - }, - "v1alpha1.GroupVersionKind": { - "description": "GroupVersionKind unambiguously identifies a kind. It doesn't anonymously include GroupVersion to avoid automatic coercion. It doesn't use a GroupVersion to avoid custom marshalling.", - "required": [ - "group", - "version", - "kind" - ], - "properties": { - "group": { - "type": "string" - }, - "kind": { - "type": "string" - }, - "version": { - "type": "string" - } - } - }, - "v1alpha1.NodeStatus": { - "description": "NodeStatus describes the status for an individual node in the sensor's FSM. A single node can represent the status for signal or a trigger.", - "required": [ - "id", - "name", - "displayName", - "type", - "phase" - ], - "properties": { - "displayName": { - "description": "DisplayName is the human readable representation of the node", - "type": "string" - }, - "id": { - "description": "ID is a unique identifier of a node within a sensor It is a hash of the node name", - "type": "string" - }, - "latestEvent": { - "description": "LatestEvent stores the last seen event for this node", - "$ref": "#/definitions/v1alpha1.EventWrapper" - }, - "message": { - "description": "store data or something to save for signal notifications or trigger events", - "type": "string" - }, - "name": { - "description": "Name is a unique name in the node tree used to generate the node ID", - "type": "string" - }, - "phase": { - "description": "Phase of the node", - "type": "string" - }, - "type": { - "description": "Type is the type of the node", - "type": "string" - } - } - }, - "v1alpha1.ResourceObject": { - "description": "ResourceObject is the resource object to create on kubernetes", - "required": [ - "group", - "version", - "kind", - "namespace", - "source", - "parameters" - ], - "properties": { - "group": { - "type": "string" - }, - "kind": { - "type": "string" - }, - "labels": { - "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. This overrides any labels in the unstructured object with the same key.", - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "namespace": { - "description": "Namespace in which to create this object optional defaults to the service account namespace", - "type": "string" - }, - "parameters": { - "description": "Parameters is the list of resource parameters to pass in the object", - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1.ResourceParameter" - } - }, - "source": { - "description": "Source of the K8 resource file(s)", - "$ref": "#/definitions/v1alpha1.ArtifactLocation" - }, - "version": { - "type": "string" - } - } - }, - "v1alpha1.ResourceParameter": { - "description": "ResourceParameter indicates a passed parameter to a service template", - "required": [ - "src", - "dest" - ], - "properties": { - "dest": { - "description": "Dest is the JSONPath of a resource key. A path is a series of keys separated by a dot. The colon character can be escaped with '.' The -1 key can be used to append a value to an existing array. See https://github.com/tidwall/sjson#path-syntax for more information about how this is used.", - "type": "string" - }, - "src": { - "description": "Src contains a source reference to the value of the resource parameter from a signal event", - "$ref": "#/definitions/v1alpha1.ResourceParameterSource" - } - } - }, - "v1alpha1.ResourceParameterSource": { - "description": "ResourceParameterSource defines the source for a resource parameter from a signal event", - "required": [ - "signal", - "path" - ], - "properties": { - "path": { - "description": "Path is the JSONPath of the event's (JSON decoded) data key Path is a series of keys separated by a dot. A key may contain wildcard characters '*' and '?'. To access an array value use the index as the key. The dot and wildcard characters can be escaped with '\\'. See https://github.com/tidwall/gjson#path-syntax for more information on how to use this.", - "type": "string" - }, - "signal": { - "description": "Signal is the name of the signal for which to retrieve this event", - "type": "string" - }, - "value": { - "description": "Value is the default literal value to use for this parameter source This is only used if the path is invalid. If the path is invalid and this is not defined, this param source will produce an error.", - "type": "string" - } - } - }, - "v1alpha1.RetryStrategy": { - "description": "RetryStrategy represents a strategy for retrying operations" - }, - "v1alpha1.S3Artifact": { - "description": "S3Artifact contains information about an artifact in S3", - "properties": { - "bucket": { - "type": "string" - }, - "endpoint": { - "type": "string" - }, - "event": { - "type": "string" - }, - "filter": { - "$ref": "#/definitions/v1alpha1.S3Filter" - }, - "insecure": { - "type": "boolean" - }, - "key": { - "type": "string" - }, - "region": { - "type": "string" - } - } - }, - "v1alpha1.S3Bucket": { - "description": "S3Bucket contains information for an S3 Bucket", - "properties": { - "bucket": { - "type": "string" - }, - "endpoint": { - "type": "string" - }, - "insecure": { - "type": "boolean" - }, - "region": { - "type": "string" - } - } - }, - "v1alpha1.S3Filter": { - "description": "S3Filter represents filters to apply to bucket nofifications for specifying constraints on objects", - "required": [ - "prefix", - "suffix" - ], - "properties": { - "prefix": { - "type": "string" - }, - "suffix": { - "type": "string" - } - } - }, - "v1alpha1.Sensor": { - "description": "Sensor is the definition of a sensor resource", - "required": [ - "spec", - "status" - ], - "properties": { - "spec": { - "$ref": "#/definitions/v1alpha1.SensorSpec" - }, - "status": { - "$ref": "#/definitions/v1alpha1.SensorStatus" - } - } - }, - "v1alpha1.SensorList": { - "description": "SensorList is the list of Sensor resources", - "required": [ - "items" - ], - "properties": { - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1.Sensor" - } - } - } - }, - "v1alpha1.SensorSpec": { - "description": "SensorSpec represents desired sensor state", - "required": [ - "signals", - "triggers" - ], - "properties": { - "imagePullPolicy": { - "description": "ImagePullPolicy determines the when the image should be pulled from docker repository", - "type": "string" - }, - "imageVersion": { - "description": "ImageVersion is the sensor image version to run", - "type": "string" - }, - "repeat": { - "description": "Repeat is a flag that determines if the sensor status should be reset after completion. NOTE: functionality is currently experimental and part of an initiative to define a more concrete pattern or cycle for sensor repetition.", - "type": "boolean" - }, - "serviceAccountName": { - "description": "ServiceAccountName required for role based access", - "type": "string" - }, - "signals": { - "description": "Signals is a list of the things that this sensor is dependent on. These are the inputs to this sensor.", - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1.Signal" - } - }, - "triggers": { - "description": "Triggers is a list of the things that this sensor evokes. These are the outputs from this sensor.", - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1.Trigger" - } - } - } - }, - "v1alpha1.SensorStatus": { - "description": "SensorStatus contains information about the status of a sensor.", - "required": [ - "phase" - ], - "properties": { - "completionCount": { - "description": "CompletionCount is the count of sensor's successful runs.", - "type": "integer", - "format": "int32" - }, - "message": { - "description": "Message is a human readable string indicating details about a sensor in its phase", - "type": "string" - }, - "nodes": { - "description": "Nodes is a mapping between a node ID and the node's status it records the states for the FSM of this sensor.", - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/v1alpha1.NodeStatus" - } - }, - "phase": { - "description": "Phase is the high-level summary of the sensor", - "type": "string" - } - } - }, - "v1alpha1.Signal": { - "description": "Signal describes a dependency", - "required": [ - "name" - ], - "properties": { - "deadline": { - "description": "Deadline is the duration in seconds after the StartedAt time of the sensor after which this signal is terminated. Note: this functionality is not yet respected, but it's theoretical behavior is as follows: This trumps the recurrence patterns of calendar signals and allows any signal to have a strict defined life. After the deadline is reached and this signal has not in a Resolved state, this signal is marked as Failed and proper escalations should proceed.", - "type": "integer", - "format": "int64" - }, - "filters": { - "description": "Filters and rules governing tolerations of success and constraints on the context and data of an event", - "$ref": "#/definitions/v1alpha1.SignalFilter" - }, - "name": { - "description": "Name is a unique name of this dependency", - "type": "string" - } - } - }, - "v1alpha1.SignalFilter": { - "description": "SignalFilter defines filters and constraints for a signal.", - "required": [ - "name" - ], - "properties": { - "context": { - "description": "Context filter constraints with escalation", - "$ref": "#/definitions/v1alpha1.EventContext" - }, - "data": { - "description": "Data filter constraints with escalation", - "$ref": "#/definitions/v1alpha1.Data" - }, - "name": { - "description": "Name is the name of signal filter", - "type": "string" - }, - "time": { - "description": "Time filter on the signal with escalation", - "$ref": "#/definitions/v1alpha1.TimeFilter" - } - } - }, - "v1alpha1.TimeFilter": { - "description": "TimeFilter describes a window in time. Filters out signal events that occur outside the time limits. In other words, only events that occur after Start and before Stop will pass this filter.", - "properties": { - "escalationPolicy": { - "description": "EscalationPolicy is the escalation to trigger in case the signal filter fails", - "$ref": "#/definitions/v1alpha1.EscalationPolicy" - }, - "start": { - "description": "Start is the beginning of a time window. Before this time, events for this signal are ignored and format is hh:mm:ss", - "type": "string" - }, - "stop": { - "description": "StopPattern is the end of a time window. After this time, events for this signal are ignored and format is hh:mm:ss", - "type": "string" - } - } - }, - "v1alpha1.Trigger": { - "description": "Trigger is an action taken, output produced, an event created, a message sent", - "required": [ - "name", - "retryStrategy" - ], - "properties": { - "message": { - "description": "Message describes a message that will be sent on a queue", - "type": "string" - }, - "name": { - "description": "Name is a unique name of the action to take", - "type": "string" - }, - "resource": { - "description": "Resource describes the resource that will be created by this action", - "$ref": "#/definitions/v1alpha1.ResourceObject" - }, - "retryStrategy": { - "description": "RetryStrategy is the strategy to retry a trigger if it fails", - "$ref": "#/definitions/v1alpha1.RetryStrategy" - } - } - }, - "v1alpha1.URI": { - "description": "URI is a Uniform Resource Identifier based on RFC 3986", - "required": [ - "scheme", - "user", - "password", - "host", - "port", - "path", - "query", - "fragment" - ], - "properties": { - "fragment": { - "type": "string" - }, - "host": { - "type": "string" - }, - "password": { - "type": "string" - }, - "path": { - "type": "string" - }, - "port": { - "type": "integer", - "format": "int32" - }, - "query": { - "type": "string" - }, - "scheme": { - "type": "string" - }, - "user": { - "type": "string" - } - } - }, - "v1alpha1.URLArtifact": { - "description": "URLArtifact contains information about an artifact at an http endpoint.", - "properties": { - "path": { - "type": "string" - }, - "verifyCert": { - "type": "boolean" - } - } - } - } - } \ No newline at end of file diff --git a/sensors/event-handler_test.go b/sensors/event-handler_test.go index 0720c2edf0..fb8e23a71d 100644 --- a/sensors/event-handler_test.go +++ b/sensors/event-handler_test.go @@ -44,9 +44,13 @@ spec: deploySpec: containers: - name: "sensor" - image: "metalgearsolid/sensor:v0.6.1" + image: "argoproj/sensor" imagePullPolicy: Always serviceAccountName: argo-events-sa + eventProtocol: + type: "HTTP" + http: + port: "9300" dependencies: - name: test-gateway:test triggers: