Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Provide scraper for Azure Storage Queues #402

Merged
merged 17 commits into from
Mar 20, 2019
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge branch 'master' into azure-queue
tomkerkhove authored Mar 19, 2019

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 5ef157c3af596f04d04693fe0229d94ff8279980
41 changes: 41 additions & 0 deletions .azure-devops/bots-scraper-ci-docker-image.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
name: $(Date:yyyyMMdd)$(Rev:.r)
resources:
- repo: self
trigger:
batch: true
branches:
include:
- master
paths:
include:
- deploy/automation/scraper/ci/*
pr: none
variables:
LogicApp.Name: 'promitor-automation-github-ci-scraper'
GitHub.Repo.Owner: 'tomkerkhove'
GitHub.Repo.Name: 'promitor'
GitHub.Bot.Name: 'promitor-bot'
pool:
vmImage: vs2017-win2016
steps:
- task: qetza.replacetokens.replacetokens-task.replacetokens@3
displayName: 'Replace tokens in tooling'
inputs:
rootDirectory: 'deploy/automation/scraper/ci/'
targetFiles: azuredeploy.parameters.json
verbosity: detailed
actionOnMissing: fail

- task: AzureResourceGroupDeployment@2
displayName: 'Deploy CI bot for Docker Hub updates'
inputs:
azureSubscription: 'Visual Studio Enterprise (0f9d7fea-99e8-4768-8672-06a28514f77e)'
resourceGroupName: promitor
location: 'West Europe'
csmFile: 'deploy/automation/scraper/ci/azuredeploy.json'
csmParametersFile: 'deploy/automation/scraper/ci/azuredeploy.parameters.json'





282 changes: 282 additions & 0 deletions deploy/automation/scraper/ci/azuredeploy.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,282 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"LogicApp.Name": {
"type": "string",
"maxLength": 80,
"metadata": {
"description": "Name of the Logic App that will handle new Docker Hub webhooks"
}
},
"GitHub.Repo.Owner": {
"type": "string",
"defaultValue": "tomkerkhove",
"metadata": {
"description": "Name of the owner for the GitHub repo"
}
},
"GitHub.Repo.Name": {
"type": "string",
"defaultValue": "promitor",
"metadata": {
"description": "Name of the GitHub repo"
}
},
"GitHub.Bot.Name": {
"type": "string",
"defaultValue": "promitor-bot",
"metadata": {
"description": "Name of the GitHub account that will act as a bot"
}
},
"GitHub.Bot.Password": {
"type": "securestring",
"metadata": {
"description": "Password to authenticate with for the GitHub account that will act as a bot"
}
}
},
"variables": {
"Alerts.ActionGroup.Name": "Promitor - Maintainers"
},
"resources": [
{
"type": "Microsoft.Logic/workflows",
"apiVersion": "2016-06-01",
"name": "[parameters('LogicApp.Name')]",
"location": "[resourceGroup().location]",
"properties": {
"definition": {
"$schema": "https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/workflowdefinition.json#",
"actions": {
"Define_GitHub_comment_text": {
"inputs": "Docker image for this PR was built and is available on [Docker Hub](@{triggerBody()?['repository']?['repo_url']}).\n\nYou can pull it locally via the CLI:\n```shell\ndocker pull @{triggerBody()?['repository']?['repo_name']}:@{triggerBody()?['push_data']?['tag']}\n```",
"runAfter": {
"Determine_GitHub_Issue_Id": [
"Succeeded"
]
},
"type": "Compose"
},
"Determine_GitHub_Issue_Id": {
"inputs": "@replace(toLower(triggerBody()?['push_data']?['tag']), 'pr', '')",
"runAfter": {},
"type": "Compose"
},
"HTTP": {
"inputs": {
"authentication": {
"password": "[parameters('GitHub.Bot.Password')]",
"type": "Basic",
"username": "[parameters('GitHub.Bot.Name')]"
},
"body": {
"body": "@{outputs('Define_GitHub_comment_text')}"
},
"method": "POST",
"uri": "@{concat('https://api.github.com/repos/', parameters('GitHub.Repo.Owner'),'/', parameters('GitHub.Repo.Name'),'/issues/', outputs('Determine_GitHub_Issue_Id'),'/comments')}"
},
"runAfter": {
"Define_GitHub_comment_text": [
"Succeeded"
]
},
"type": "Http"
}
},
"contentVersion": "1.0.0.0",
"outputs": {},
"parameters": {
"GitHub.Repo.Owner": {
"defaultValue": "[parameters('GitHub.Repo.Owner')]",
"type": "string"
},
"GitHub.Repo.Name": {
"defaultValue": "[parameters('GitHub.Repo.Name')]",
"type": "string"
}
},
"triggers": {
"manual": {
"inputs": {
"schema": {
"properties": {
"callback_url": {
"type": "string"
},
"push_data": {
"properties": {
"images": {
"type": "array"
},
"pushed_at": {
"type": "integer"
},
"pusher": {
"type": "string"
},
"tag": {
"type": "string"
}
},
"type": "object"
},
"repository": {
"properties": {
"comment_count": {
"type": "integer"
},
"date_created": {
"type": "integer"
},
"description": {
"type": "string"
},
"full_description": {
"type": "string"
},
"is_official": {
"type": "boolean"
},
"is_private": {
"type": "boolean"
},
"is_trusted": {
"type": "boolean"
},
"name": {
"type": "string"
},
"namespace": {
"type": "string"
},
"owner": {
"type": "string"
},
"repo_name": {
"type": "string"
},
"repo_url": {
"type": "string"
},
"star_count": {
"type": "integer"
},
"status": {
"type": "string"
}
},
"type": "object"
}
},
"type": "object"
}
},
"kind": "Http",
"type": "Request"
}
}
},
"parameters": {}
},
"dependsOn": []
},
{
"type": "Microsoft.Insights/actionGroups",
"apiVersion": "2018-03-01",
"name": "[variables('Alerts.ActionGroup.Name')]",
"location": "Global",
"properties": {
"groupShortName": "AlertHandler",
"enabled": true,
"smsReceivers": [],
"emailReceivers": [
{
"name": "Tom Kerkhove",
"emailAddress": "kerkhove.tom@gmail.com"
}
],
"webhookReceivers": []
}
},
{
"name": "Scraper CI Bot - Docker Hub Integration Run Failed",
"type": "Microsoft.Insights/metricAlerts",
"location": "global",
"apiVersion": "2018-03-01",
"tags": {},
"properties": {
"description": "Scraper CI Bot - Docker Hub integration failed to run and post a comment on the pull request in GitHub",
"severity": 3,
"enabled": true,
"scopes": [
"[resourceId('Microsoft.Logic/workflows', parameters('LogicApp.Name'))]"
],
"evaluationFrequency": "PT5M",
"windowSize": "PT5M",
"criteria": {
"odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria",
"allOf": [
{
"name": "1st criterion",
"metricName": "RunsFailed",
"dimensions": [],
"operator": "GreaterThanOrEqual",
"threshold": "1",
"timeAggregation": "Total"
}
]
},
"actions": [
{
"actionGroupId": "[resourceId('Microsoft.Insights/actionGroups', variables('Alerts.ActionGroup.Name'))]"
}
]
},
"dependsOn": [
"[resourceId('Microsoft.Insights/actionGroups', variables('Alerts.ActionGroup.Name'))]",
"[resourceId('Microsoft.Logic/workflows', parameters('LogicApp.Name'))]"
]
},
{
"name": "Scraper CI Bot - Docker Hub Integration Trigger Failed",
"type": "Microsoft.Insights/metricAlerts",
"location": "global",
"apiVersion": "2018-03-01",
"tags": {},
"properties": {
"description": "Scraper CI Bot - Docker Hub integration failed to be triggered due to an issue",
"severity": 3,
"enabled": true,
"scopes": [
"[resourceId('Microsoft.Logic/workflows', parameters('LogicApp.Name'))]"
],
"evaluationFrequency": "PT5M",
"windowSize": "PT5M",
"criteria": {
"odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria",
"allOf": [
{
"name": "1st criterion",
"metricName": "TriggersFailed",
"dimensions": [],
"operator": "GreaterThanOrEqual",
"threshold": "1",
"timeAggregation": "Total"
}
]
},
"actions": [
{
"actionGroupId": "[resourceId('Microsoft.Insights/actionGroups', variables('Alerts.ActionGroup.Name'))]"
}
]
},
"dependsOn": [
"[resourceId('Microsoft.Insights/actionGroups', variables('Alerts.ActionGroup.Name'))]",
"[resourceId('Microsoft.Logic/workflows', parameters('LogicApp.Name'))]"
]
}
]
}
21 changes: 21 additions & 0 deletions deploy/automation/scraper/ci/azuredeploy.parameters.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json",
"contentVersion": "1.0.0.0",
"parameters": {
"LogicApp.Name": {
"value": "#{LogicApp.Name}#"
},
"GitHub.Repo.Owner": {
"value": "#{GitHub.Repo.Owner}#"
},
"GitHub.Repo.Name": {
"value": "#{GitHub.Repo.Name}#"
},
"GitHub.Bot.Name": {
"value": "#{GitHub.Bot.Name}#"
},
"GitHub.Bot.Password": {
"value": "#{GitHub.Bot.Password}#"
}
}
}
2 changes: 1 addition & 1 deletion docs/configuration/index.md
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ Promitor automatically scrapes Azure Monitor and makes the information available
The behavior of this can be configured with the following environment variables:
- **PROMITOR_CONFIGURATION_PATH** - Defines the location of the YAML file that declares what Azure Monitor metrics to scrape. If nothing is specified, `/config/metrics-declaration.yaml` will be used.
- **PROMITOR_SCRAPE_BASEPATH** - Controls the path where the scraping endpoint for Prometheus is being exposed. If nothing is specified, `/prometheus/scrape` will be used.
- **PROMITOR_SCRAPE_SCHEDULE** - A cron expression that controls the fequency in which all the configured metrics will be scraped from Azure Monitor. If configured is specified, `*/5 * * * *` will be used.
- **PROMITOR_SCRAPE_SCHEDULE** - A cron expression that controls the fequency in which all the configured metrics will be scraped from Azure Monitor. You can use [crontab-generator.org](https://crontab-generator.org/) to generate a cron that fits your needs. If nothing is configured, `*/5 * * * *` will be used.

We're also providing feature flags to opt-out of certain features:
- **PROMITOR_FEATURE_METRICSTIMESTAMP** - Defines whether or not a timestamp should be included when the value was scraped on Azure Monitor. Supported values are `True` to opt-in & `False` to opt-out, if nothing is configured this will be turned on.
3 changes: 2 additions & 1 deletion docs/thank-you.md
Original file line number Diff line number Diff line change
@@ -39,10 +39,11 @@ We are using the following GitHub Apps:

# NuGet Packages
Here is an overview of the NuGet packages that we rely on:
- [CronScheduler.AspNetCore](https://github.com/kdcllc/CronScheduler.AspNetCore) - Asp.Net Core 2.x Hosted or .Net Core 2.x Self-hosted Cron Scheduler
- [Swashbuckle.AspNetCore](https://github.com/domaindrivendev/Swashbuckle.AspNetCore) - Swagger tools for documenting API's built on ASP.NET Core
- [Prometheus.Client](https://github.com/PrometheusClientNet/Prometheus.Client) - .NET client for prometheus.io
- [Shuttle.Core.Cron](https://github.com/Shuttle/Shuttle.Core.Cron) - Cron expression parsing
- [YamlDotNet](https://github.com/aaubry/YamlDotNet) - .NET library for YAML
- [Guard.NET](https://github.com/george-pancescu/Guard) - Library that facilitates runtime checks of code and allows to define preconditions and invariants within a method
- [Cronos](https://github.com/HangfireIO/Cronos) - Fully-featured .NET library for working with Cron expressions. Built with time zones in mind and intuitively handles daylight saving time transitions
- [Bogus](https://github.com/bchavez/Bogus) - A simple and sane data generator for populating objects
- [xUnit](https://github.com/xunit/xunit) - Free, open source, community-focused unit testing tool for the .NET
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResouceTypes
namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes
{
public class GenericMetricDefinition : MetricDefinition
{
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResouceTypes
namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes
{
public class ServiceBusQueueMetricDefinition : MetricDefinition
{
Original file line number Diff line number Diff line change
@@ -26,8 +26,7 @@ internal List<TObject> Deserialize(YamlSequenceNode nodes)
var deserializedObjects = new List<TObject>();
foreach (var item in nodes)
{
var metricNode = item as YamlMappingNode;
if (metricNode == null)
if (!(item is YamlMappingNode metricNode))
{
throw new SerializationException($"Failed parsing metrics because we couldn't cast an item to {nameof(YamlMappingNode)}");
}
Loading
You are viewing a condensed version of this merge commit. You can view the full changes here.