Merge pull request #697 from NixOS/feat/webhook-riir

Rewrite the webhook receiver in Rust

.github/workflows/ci.yml

@@ -30,4 +30,4 @@ jobs:
         with:
           nix_path: nixpkgs=channel:nixos-unstable
       - name: nix-build
-        run: nix-build -A ofborg.rs -A ofborg.php
+        run: nix-build -A ofborg.rs

.gitignore

@@ -1,7 +1,5 @@
-config.php
 vendor
 *.log
-test.php
 config.json
 .bash_hist
 config.private.json

config.public.json

@@ -1,4 +1,15 @@
 {
+    "github_webhook_receiver": {
+        "listen": "[::1]:9899",
+        "webhook_secret_file: "/run/secrets/ofborg/github-webhook-secret",
+        "rabbitmq": {
+            "host": "localhost",
+            "ssl": false,
+            "username": "ofborg-github-webhook",
+            "password_file": "/run/secrets/ofborg/github-webhook-amqp-password",
+            "virtualhost": "ofborg"
+        }
+    },
     "feedback": {
         "full_logs": true
     },

flake.nix

@@ -1,13 +1,11 @@
 {
   inputs = {
-    nixpkgs.url = "github:nixos/nixpkgs/nixos-22.11";
-    nixpkgs-for-php.url = "github:nixos/nixpkgs/nixos-22.05";
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11";
   };
 
   outputs =
     { self
     , nixpkgs
-    , nixpkgs-for-php
     , ...
     }@inputs:
     let
@@ -22,21 +20,6 @@
             pkgs = import nixpkgs {
               inherit system;
             };
-            phpPkgs = import nixpkgs-for-php {
-              inherit system;
-            };
-
-            phpEnv = pkgs.mkShell {
-              name = "gh-event-forwarder";
-              buildInputs = with pkgs; [
-                nix-prefetch-git
-                phpPkgs.php
-                phpPkgs.phpPackages.composer
-                git
-                curl
-                bash
-              ];
-            };
           in
           {
             default = pkgs.mkShell {
@@ -78,7 +61,6 @@
               RUST_BACKTRACE = "1";
               RUST_LOG = "ofborg=debug";
               NIX_PATH = "nixpkgs=${pkgs.path}";
-              passthru.phpEnv = phpEnv;
             };
           });
 
@@ -88,16 +70,12 @@
             inherit system;
           };
 
-          phpPkgs = import nixpkgs-for-php {
-            inherit system;
-          };
-
           pkg = pkgs.rustPlatform.buildRustPackage {
             name = "ofborg";
             src = pkgs.nix-gitignore.gitignoreSource [ ] ./.;
 
             nativeBuildInputs = with pkgs; [
-              pkgconfig
+              pkg-config
               pkgs.rustPackages.clippy
             ];
 
@@ -144,16 +122,14 @@
             test -e $out/bin/builder
             test -e $out/bin/github_comment_filter
             test -e $out/bin/github_comment_poster
+            test -e $out/bin/github_webhook_receiver
             test -e $out/bin/log_message_collector
             test -e $out/bin/evaluation_filter
           '';
-
-          ofborg.php = import ./php { pkgs = phpPkgs; };
         });
 
       hydraJobs = {
         buildRs = forAllSystems (system: self.packages.${system}.ofborg.rs);
-        buildPhp = self.packages.x86_64-linux.ofborg.php;
       };
     };
 }

ofborg/Cargo.toml

@@ -34,3 +34,6 @@ tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.16", features = ["json", "env-filter"] }
 uuid = { version = "1.2", features = ["v4"] }
 rustls-pemfile = "1.0.2"
+hmac = "0.12.1"
+sha2 = "0.10.8"
+hex = "0.4.3"

ofborg/src/bin/evaluation-filter.rs

@@ -2,7 +2,7 @@ use std::env;
 use std::error::Error;
 
 use async_std::task;
-use tracing::info;
+use tracing::{error, info};
 
 use ofborg::config;
 use ofborg::easyamqp::{self, ChannelExt, ConsumerExt};
@@ -14,10 +14,15 @@ fn main() -> Result<(), Box<dyn Error>> {
 
     let arg = env::args()
         .nth(1)
-        .expect("usage: evaluation-filter <config>");
+        .unwrap_or_else(|| panic!("usage: {} <config>", std::env::args().next().unwrap()));
     let cfg = config::load(arg.as_ref());
 
-    let conn = easylapin::from_config(&cfg.rabbitmq)?;
+    let Some(filter_cfg) = config::load(arg.as_ref()).evaluation_filter else {
+        error!("No evaluation filter configuration found!");
+        panic!();
+    };
+
+    let conn = easylapin::from_config(&filter_cfg.rabbitmq)?;
     let mut chan = task::block_on(conn.create_channel())?;
 
     chan.declare_exchange(easyamqp::ExchangeConfig {
@@ -52,7 +57,7 @@ fn main() -> Result<(), Box<dyn Error>> {
     chan.bind_queue(easyamqp::BindQueueConfig {
         queue: queue_name.clone(),
         exchange: "github-events".to_owned(),
-        routing_key: Some("pull_request.nixos/nixpkgs".to_owned()),
+        routing_key: Some("pull_request.nixos/*".to_owned()),
         no_wait: false,
     })?;
 

ofborg/src/bin/github-comment-filter.rs

@@ -2,7 +2,8 @@ use std::env;
 use std::error::Error;
 
 use async_std::task;
-use tracing::info;
+use ofborg::systems::System;
+use tracing::{error, info};
 
 use ofborg::config;
 use ofborg::easyamqp::{self, ChannelExt, ConsumerExt};
@@ -14,10 +15,15 @@ fn main() -> Result<(), Box<dyn Error>> {
 
     let arg = env::args()
         .nth(1)
-        .expect("usage: github-comment-filter <config>");
+        .unwrap_or_else(|| panic!("usage: {} <config>", std::env::args().next().unwrap()));
     let cfg = config::load(arg.as_ref());
 
-    let conn = easylapin::from_config(&cfg.rabbitmq)?;
+    let Some(filter_cfg) = config::load(arg.as_ref()).github_comment_filter else {
+        error!("No comment filter configuration found!");
+        panic!();
+    };
+
+    let conn = easylapin::from_config(&filter_cfg.rabbitmq)?;
     let mut chan = task::block_on(conn.create_channel())?;
 
     chan.declare_exchange(easyamqp::ExchangeConfig {
@@ -57,6 +63,28 @@ fn main() -> Result<(), Box<dyn Error>> {
         no_wait: false,
     })?;
 
+    chan.declare_exchange(easyamqp::ExchangeConfig {
+        exchange: "build-results".to_owned(),
+        exchange_type: easyamqp::ExchangeType::Fanout,
+        passive: false,
+        durable: true,
+        auto_delete: false,
+        no_wait: false,
+        internal: false,
+    })?;
+
+    // Create build job queues
+    for sys in System::all_known_systems().iter().map(System::to_string) {
+        chan.declare_queue(easyamqp::QueueConfig {
+            queue: format!("build-inputs-{sys}"),
+            passive: false,
+            durable: true,
+            exclusive: false,
+            auto_delete: false,
+            no_wait: false,
+        })?;
+    }
+
     let handle = easylapin::WorkerChannel(chan).consume(
         tasks::githubcommentfilter::GitHubCommentWorker::new(cfg.acl(), cfg.github()),
         easyamqp::ConsumeConfig {