From 059463bbe1c3ac62680e29a78d5bb4655fa92553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Mon, 23 Dec 2024 20:32:25 +0100 Subject: [PATCH 01/10] Drop all webhook receiver PHP code --- .github/workflows/ci.yml | 2 +- .gitignore | 2 - flake.lock | 19 +- flake.nix | 25 --- php/README.md | 38 ---- php/composer-env.nix | 244 ----------------------- php/composer.json | 7 - php/composer.lock | 412 --------------------------------------- php/default.nix | 14 -- php/php-packages.nix | 75 ------- php/web/index.php | 186 ------------------ 11 files changed, 2 insertions(+), 1022 deletions(-) delete mode 100644 php/README.md delete mode 100644 php/composer-env.nix delete mode 100644 php/composer.json delete mode 100644 php/composer.lock delete mode 100644 php/default.nix delete mode 100644 php/php-packages.nix delete mode 100644 php/web/index.php diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2de92e43..8f71a46e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,4 +30,4 @@ jobs: with: nix_path: nixpkgs=channel:nixos-unstable - name: nix-build - run: nix-build -A ofborg.rs -A ofborg.php + run: nix-build -A ofborg.rs diff --git a/.gitignore b/.gitignore index eeceedea..edf0ba83 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ -config.php vendor *.log -test.php config.json .bash_hist config.private.json diff --git a/flake.lock b/flake.lock index abb3bb56..1e6d6eed 100644 --- a/flake.lock +++ b/flake.lock @@ -16,26 +16,9 @@ "type": "github" } }, - "nixpkgs-for-php": { - "locked": { - "lastModified": 1670538458, - "narHash": "sha256-mvKmBkdlhzsMBtnzYXjYn08EGw9rFBEE9hp4Uqgol1Q=", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "99ec06122f481588abafd91f2710d80a5320efe6", - "type": "github" - }, - "original": { - "owner": "nixos", - "ref": "nixos-22.05", - "repo": "nixpkgs", - "type": "github" - } - }, "root": { "inputs": { - "nixpkgs": "nixpkgs", - "nixpkgs-for-php": "nixpkgs-for-php" + "nixpkgs": "nixpkgs" } } }, diff --git a/flake.nix b/flake.nix index e4e18060..b4a11d85 100644 --- a/flake.nix +++ b/flake.nix @@ -1,13 +1,11 @@ { inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixos-22.11"; - nixpkgs-for-php.url = "github:nixos/nixpkgs/nixos-22.05"; }; outputs = { self , nixpkgs - , nixpkgs-for-php , ... }@inputs: let @@ -22,21 +20,6 @@ pkgs = import nixpkgs { inherit system; }; - phpPkgs = import nixpkgs-for-php { - inherit system; - }; - - phpEnv = pkgs.mkShell { - name = "gh-event-forwarder"; - buildInputs = with pkgs; [ - nix-prefetch-git - phpPkgs.php - phpPkgs.phpPackages.composer - git - curl - bash - ]; - }; in { default = pkgs.mkShell { @@ -78,7 +61,6 @@ RUST_BACKTRACE = "1"; RUST_LOG = "ofborg=debug"; NIX_PATH = "nixpkgs=${pkgs.path}"; - passthru.phpEnv = phpEnv; }; }); @@ -88,10 +70,6 @@ inherit system; }; - phpPkgs = import nixpkgs-for-php { - inherit system; - }; - pkg = pkgs.rustPlatform.buildRustPackage { name = "ofborg"; src = pkgs.nix-gitignore.gitignoreSource [ ] ./.; @@ -147,13 +125,10 @@ test -e $out/bin/log_message_collector test -e $out/bin/evaluation_filter ''; - - ofborg.php = import ./php { pkgs = phpPkgs; }; }); hydraJobs = { buildRs = forAllSystems (system: self.packages.${system}.ofborg.rs); - buildPhp = self.packages.x86_64-linux.ofborg.php; }; }; } diff --git a/php/README.md b/php/README.md deleted file mode 100644 index 4bf1bac4..00000000 --- a/php/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# Webhook Receiver - -This PHP code receives the GitHub webhook, checks them for integrity and publishes messages on rabbitmq. - - -## Configuration - -The code expects a `config.php` in it's parent directory. An example configuration looks like this: - -```php - true, - 'verify_peer_name' => true, - 'peer_name' => $host, - 'verify_depth' => 10, - 'ca_file' => '/etc/ssl/certs/ca-certificates.crt', - ), array( - 'connection_timeout' => $timeout, - ) - ); - return $connection; -} - -function gh_secret() { - return "github webhook secret"; -} -``` diff --git a/php/composer-env.nix b/php/composer-env.nix deleted file mode 100644 index 71714b76..00000000 --- a/php/composer-env.nix +++ /dev/null @@ -1,244 +0,0 @@ -# This file originates from composer2nix - -{ stdenv, lib, writeTextFile, fetchurl, php, unzip, phpPackages }: - -let - inherit (phpPackages) composer; - - filterSrc = src: - builtins.filterSource (path: type: type != "directory" || (baseNameOf path != ".git" && baseNameOf path != ".git" && baseNameOf path != ".svn")) src; - - buildZipPackage = { name, src }: - stdenv.mkDerivation { - inherit name src; - nativeBuildInputs = [ unzip ]; - buildCommand = '' - shopt -s dotglob - unzip $src - baseDir=$(find . -type d -mindepth 1 -maxdepth 1) - cd $baseDir - mkdir -p $out - mv * $out - ''; - }; - - buildPackage = - { name - , src - , packages ? {} - , devPackages ? {} - , buildInputs ? [] - , symlinkDependencies ? false - , executable ? false - , removeComposerArtifacts ? false - , postInstall ? "" - , noDev ? false - , composerExtraArgs ? "" - , unpackPhase ? "true" - , buildPhase ? "true" - , ...}@args: - - let - reconstructInstalled = writeTextFile { - name = "reconstructinstalled.php"; - executable = true; - text = '' - #! ${php}/bin/php - - ''; - }; - - constructBin = writeTextFile { - name = "constructbin.php"; - executable = true; - text = '' - #! ${php}/bin/php - - ''; - }; - - bundleDependencies = dependencies: - lib.concatMapStrings (dependencyName: - let - dependency = dependencies.${dependencyName}; - in - '' - ${if dependency.targetDir == "" then '' - vendorDir="$(dirname ${dependencyName})" - mkdir -p "$vendorDir" - ${if symlinkDependencies then - ''ln -s "${dependency.src}" "$vendorDir/$(basename "${dependencyName}")"'' - else - ''cp -av "${dependency.src}" "$vendorDir/$(basename "${dependencyName}")"'' - } - '' else '' - namespaceDir="${dependencyName}/$(dirname "${dependency.targetDir}")" - mkdir -p "$namespaceDir" - ${if symlinkDependencies then - ''ln -s "${dependency.src}" "$namespaceDir/$(basename "${dependency.targetDir}")"'' - else - ''cp -av "${dependency.src}" "$namespaceDir/$(basename "${dependency.targetDir}")"'' - } - ''} - '') (builtins.attrNames dependencies); - - extraArgs = removeAttrs args [ "packages" "devPackages" "buildInputs" ]; - in - stdenv.mkDerivation ({ - buildInputs = [ php composer ] ++ buildInputs; - - inherit unpackPhase buildPhase; - - installPhase = '' - ${if executable then '' - mkdir -p $out/share/php - cp -av $src $out/share/php/$name - chmod -R u+w $out/share/php/$name - cd $out/share/php/$name - '' else '' - cp -av $src $out - chmod -R u+w $out - cd $out - ''} - - # Remove unwanted files - rm -f *.nix - - export HOME=$TMPDIR - - # Remove the provided vendor folder if it exists - rm -Rf vendor - - # If there is no composer.lock file, compose a dummy file. - # Otherwise, composer attempts to download the package.json file from - # the registry which we do not want. - if [ ! -f composer.lock ] - then - cat > composer.lock < vendor/composer/installed.json - - # Copy or symlink the provided dependencies - cd vendor - ${bundleDependencies packages} - ${lib.optionalString (!noDev) (bundleDependencies devPackages)} - cd .. - - # Reconstruct autoload scripts - # We use the optimize feature because Nix packages cannot change after they have been built - # Using the dynamic loader for a Nix package is useless since there is nothing to dynamically reload. - composer dump-autoload --optimize ${lib.optionalString noDev "--no-dev"} ${composerExtraArgs} - - # Run the install step as a validation to confirm that everything works out as expected - composer install --optimize-autoloader ${lib.optionalString noDev "--no-dev"} ${composerExtraArgs} - - ${lib.optionalString executable '' - # Reconstruct the bin/ folder if we deploy an executable project - ${php}/bin/php ${constructBin} composer.json - ln -s $(pwd)/vendor/bin $out/bin - ''} - - ${lib.optionalString (!symlinkDependencies) '' - # Patch the shebangs if possible - if [ -d $(pwd)/vendor/bin ] - then - # Look for all executables in bin/ - for i in $(pwd)/vendor/bin/* - do - # Look for their location - realFile=$(readlink -f "$i") - - # Restore write permissions - chmod u+wx "$(dirname "$realFile")" - chmod u+w "$realFile" - - # Patch shebang - sed -e "s|#!/usr/bin/php|#!${php}/bin/php|" \ - -e "s|#!/usr/bin/env php|#!${php}/bin/php|" \ - "$realFile" > tmp - mv tmp "$realFile" - chmod u+x "$realFile" - done - fi - ''} - - if [ "$removeComposerArtifacts" = "1" ] - then - # Remove composer stuff - rm -f composer.json composer.lock - fi - - # Execute post install hook - runHook postInstall - ''; - } // extraArgs); -in -{ - inherit filterSrc; - composer = lib.makeOverridable composer; - buildZipPackage = lib.makeOverridable buildZipPackage; - buildPackage = lib.makeOverridable buildPackage; -} diff --git a/php/composer.json b/php/composer.json deleted file mode 100644 index bd1bd169..00000000 --- a/php/composer.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "nixos/ofborg-webhook", - "require": { - "php-amqplib/php-amqplib": ">=3.6.2", - "svanderburg/composer2nix": ">=0.0.6" - } -} diff --git a/php/composer.lock b/php/composer.lock deleted file mode 100644 index fdd18ef2..00000000 --- a/php/composer.lock +++ /dev/null @@ -1,412 +0,0 @@ -{ - "_readme": [ - "This file locks the dependencies of your project to a known state", - "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", - "This file is @generated automatically" - ], - "content-hash": "cddc3bf153efe4b7d2314531fd7850a3", - "packages": [ - { - "name": "paragonie/constant_time_encoding", - "version": "v2.7.0", - "source": { - "type": "git", - "url": "https://github.com/paragonie/constant_time_encoding.git", - "reference": "52a0d99e69f56b9ec27ace92ba56897fe6993105" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/52a0d99e69f56b9ec27ace92ba56897fe6993105", - "reference": "52a0d99e69f56b9ec27ace92ba56897fe6993105", - "shasum": "" - }, - "require": { - "php": "^7|^8" - }, - "require-dev": { - "phpunit/phpunit": "^6|^7|^8|^9", - "vimeo/psalm": "^1|^2|^3|^4" - }, - "type": "library", - "autoload": { - "psr-4": { - "ParagonIE\\ConstantTime\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Paragon Initiative Enterprises", - "email": "security@paragonie.com", - "homepage": "https://paragonie.com", - "role": "Maintainer" - }, - { - "name": "Steve 'Sc00bz' Thomas", - "email": "steve@tobtu.com", - "homepage": "https://www.tobtu.com", - "role": "Original Developer" - } - ], - "description": "Constant-time Implementations of RFC 4648 Encoding (Base-64, Base-32, Base-16)", - "keywords": [ - "base16", - "base32", - "base32_decode", - "base32_encode", - "base64", - "base64_decode", - "base64_encode", - "bin2hex", - "encoding", - "hex", - "hex2bin", - "rfc4648" - ], - "support": { - "email": "info@paragonie.com", - "issues": "https://github.com/paragonie/constant_time_encoding/issues", - "source": "https://github.com/paragonie/constant_time_encoding" - }, - "time": "2024-05-08T12:18:48+00:00" - }, - { - "name": "paragonie/random_compat", - "version": "v9.99.100", - "source": { - "type": "git", - "url": "https://github.com/paragonie/random_compat.git", - "reference": "996434e5492cb4c3edcb9168db6fbb1359ef965a" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/paragonie/random_compat/zipball/996434e5492cb4c3edcb9168db6fbb1359ef965a", - "reference": "996434e5492cb4c3edcb9168db6fbb1359ef965a", - "shasum": "" - }, - "require": { - "php": ">= 7" - }, - "require-dev": { - "phpunit/phpunit": "4.*|5.*", - "vimeo/psalm": "^1" - }, - "suggest": { - "ext-libsodium": "Provides a modern crypto API that can be used to generate random bytes." - }, - "type": "library", - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Paragon Initiative Enterprises", - "email": "security@paragonie.com", - "homepage": "https://paragonie.com" - } - ], - "description": "PHP 5.x polyfill for random_bytes() and random_int() from PHP 7", - "keywords": [ - "csprng", - "polyfill", - "pseudorandom", - "random" - ], - "support": { - "email": "info@paragonie.com", - "issues": "https://github.com/paragonie/random_compat/issues", - "source": "https://github.com/paragonie/random_compat" - }, - "time": "2020-10-15T08:29:30+00:00" - }, - { - "name": "php-amqplib/php-amqplib", - "version": "v3.6.2", - "source": { - "type": "git", - "url": "https://github.com/php-amqplib/php-amqplib.git", - "reference": "cb514530ce45a6d2f636be5196010c47c3bcf6e0" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/php-amqplib/php-amqplib/zipball/cb514530ce45a6d2f636be5196010c47c3bcf6e0", - "reference": "cb514530ce45a6d2f636be5196010c47c3bcf6e0", - "shasum": "" - }, - "require": { - "ext-mbstring": "*", - "ext-sockets": "*", - "php": "^7.2||^8.0", - "phpseclib/phpseclib": "^2.0|^3.0" - }, - "conflict": { - "php": "7.4.0 - 7.4.1" - }, - "replace": { - "videlalvaro/php-amqplib": "self.version" - }, - "require-dev": { - "ext-curl": "*", - "nategood/httpful": "^0.2.20", - "phpunit/phpunit": "^7.5|^9.5", - "squizlabs/php_codesniffer": "^3.6" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "3.0-dev" - } - }, - "autoload": { - "psr-4": { - "PhpAmqpLib\\": "PhpAmqpLib/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "LGPL-2.1-or-later" - ], - "authors": [ - { - "name": "Alvaro Videla", - "role": "Original Maintainer" - }, - { - "name": "Raúl Araya", - "email": "nubeiro@gmail.com", - "role": "Maintainer" - }, - { - "name": "Luke Bakken", - "email": "luke@bakken.io", - "role": "Maintainer" - }, - { - "name": "Ramūnas Dronga", - "email": "github@ramuno.lt", - "role": "Maintainer" - } - ], - "description": "Formerly videlalvaro/php-amqplib. This library is a pure PHP implementation of the AMQP protocol. It's been tested against RabbitMQ.", - "homepage": "https://github.com/php-amqplib/php-amqplib/", - "keywords": [ - "message", - "queue", - "rabbitmq" - ], - "support": { - "issues": "https://github.com/php-amqplib/php-amqplib/issues", - "source": "https://github.com/php-amqplib/php-amqplib/tree/v3.6.2" - }, - "time": "2024-04-15T18:31:22+00:00" - }, - { - "name": "phpseclib/phpseclib", - "version": "3.0.37", - "source": { - "type": "git", - "url": "https://github.com/phpseclib/phpseclib.git", - "reference": "cfa2013d0f68c062055180dd4328cc8b9d1f30b8" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/phpseclib/phpseclib/zipball/cfa2013d0f68c062055180dd4328cc8b9d1f30b8", - "reference": "cfa2013d0f68c062055180dd4328cc8b9d1f30b8", - "shasum": "" - }, - "require": { - "paragonie/constant_time_encoding": "^1|^2", - "paragonie/random_compat": "^1.4|^2.0|^9.99.99", - "php": ">=5.6.1" - }, - "require-dev": { - "phpunit/phpunit": "*" - }, - "suggest": { - "ext-dom": "Install the DOM extension to load XML formatted public keys.", - "ext-gmp": "Install the GMP (GNU Multiple Precision) extension in order to speed up arbitrary precision integer arithmetic operations.", - "ext-libsodium": "SSH2/SFTP can make use of some algorithms provided by the libsodium-php extension.", - "ext-mcrypt": "Install the Mcrypt extension in order to speed up a few other cryptographic operations.", - "ext-openssl": "Install the OpenSSL extension in order to speed up a wide variety of cryptographic operations." - }, - "type": "library", - "autoload": { - "files": [ - "phpseclib/bootstrap.php" - ], - "psr-4": { - "phpseclib3\\": "phpseclib/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Jim Wigginton", - "email": "terrafrost@php.net", - "role": "Lead Developer" - }, - { - "name": "Patrick Monnerat", - "email": "pm@datasphere.ch", - "role": "Developer" - }, - { - "name": "Andreas Fischer", - "email": "bantu@phpbb.com", - "role": "Developer" - }, - { - "name": "Hans-Jürgen Petrich", - "email": "petrich@tronic-media.com", - "role": "Developer" - }, - { - "name": "Graham Campbell", - "email": "graham@alt-three.com", - "role": "Developer" - } - ], - "description": "PHP Secure Communications Library - Pure-PHP implementations of RSA, AES, SSH2, SFTP, X.509 etc.", - "homepage": "http://phpseclib.sourceforge.net", - "keywords": [ - "BigInteger", - "aes", - "asn.1", - "asn1", - "blowfish", - "crypto", - "cryptography", - "encryption", - "rsa", - "security", - "sftp", - "signature", - "signing", - "ssh", - "twofish", - "x.509", - "x509" - ], - "support": { - "issues": "https://github.com/phpseclib/phpseclib/issues", - "source": "https://github.com/phpseclib/phpseclib/tree/3.0.37" - }, - "funding": [ - { - "url": "https://github.com/terrafrost", - "type": "github" - }, - { - "url": "https://www.patreon.com/phpseclib", - "type": "patreon" - }, - { - "url": "https://tidelift.com/funding/github/packagist/phpseclib/phpseclib", - "type": "tidelift" - } - ], - "time": "2024-03-03T02:14:58+00:00" - }, - { - "name": "svanderburg/composer2nix", - "version": "v0.0.6", - "source": { - "type": "git", - "url": "https://github.com/svanderburg/composer2nix.git", - "reference": "299caca4aac42d7639a42eb4dde951c010f6e91c" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/svanderburg/composer2nix/zipball/299caca4aac42d7639a42eb4dde951c010f6e91c", - "reference": "299caca4aac42d7639a42eb4dde951c010f6e91c", - "shasum": "" - }, - "require": { - "svanderburg/pndp": "0.0.4" - }, - "bin": [ - "bin/composer2nix" - ], - "type": "library", - "autoload": { - "psr-4": { - "Composer2Nix\\": "src/Composer2Nix" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Sander van der Burg", - "email": "svanderburg@gmail.com", - "homepage": "http://sandervanderburg.nl" - } - ], - "description": "Generate Nix expressions to build PHP composer packages", - "support": { - "issues": "https://github.com/svanderburg/composer2nix/issues", - "source": "https://github.com/svanderburg/composer2nix/tree/v0.0.6" - }, - "time": "2022-03-01T23:41:50+00:00" - }, - { - "name": "svanderburg/pndp", - "version": "v0.0.4", - "source": { - "type": "git", - "url": "https://github.com/svanderburg/pndp.git", - "reference": "bc795b341d95c24bb577e0d7a4a37fde98b1cce8" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/svanderburg/pndp/zipball/bc795b341d95c24bb577e0d7a4a37fde98b1cce8", - "reference": "bc795b341d95c24bb577e0d7a4a37fde98b1cce8", - "shasum": "" - }, - "bin": [ - "bin/pndp-build" - ], - "type": "library", - "autoload": { - "psr-4": { - "PNDP\\": "src/PNDP" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Sander van der Burg", - "email": "svanderburg@gmail.com", - "homepage": "http://sandervanderburg.nl" - } - ], - "description": "PNDP: An internal DSL for Nix in PHP", - "support": { - "issues": "https://github.com/svanderburg/pndp/issues", - "source": "https://github.com/svanderburg/pndp/tree/v0.0.4" - }, - "time": "2022-02-26T22:15:06+00:00" - } - ], - "packages-dev": [], - "aliases": [], - "minimum-stability": "stable", - "stability-flags": [], - "prefer-stable": false, - "prefer-lowest": false, - "platform": [], - "platform-dev": [], - "plugin-api-version": "2.6.0" -} diff --git a/php/default.nix b/php/default.nix deleted file mode 100644 index 2519bd84..00000000 --- a/php/default.nix +++ /dev/null @@ -1,14 +0,0 @@ -{pkgs ? import { - inherit system; - }, system ? builtins.currentSystem, noDev ? false, php ? pkgs.php, phpPackages ? pkgs.phpPackages}: - -let - composerEnv = import ./composer-env.nix { - inherit (pkgs) stdenv lib writeTextFile fetchurl unzip; - inherit php phpPackages; - }; -in -import ./php-packages.nix { - inherit composerEnv noDev; - inherit (pkgs) fetchurl fetchgit fetchhg fetchsvn; -} diff --git a/php/php-packages.nix b/php/php-packages.nix deleted file mode 100644 index fc64b5a8..00000000 --- a/php/php-packages.nix +++ /dev/null @@ -1,75 +0,0 @@ -{composerEnv, fetchurl, fetchgit ? null, fetchhg ? null, fetchsvn ? null, noDev ? false}: - -let - packages = { - "paragonie/constant_time_encoding" = { - targetDir = ""; - src = composerEnv.buildZipPackage { - name = "paragonie-constant_time_encoding-52a0d99e69f56b9ec27ace92ba56897fe6993105"; - src = fetchurl { - url = "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/52a0d99e69f56b9ec27ace92ba56897fe6993105"; - sha256 = "1ja5b3fm5v665igrd37vs28zdipbh1xgh57lil2iaggvh1b8kh4x"; - }; - }; - }; - "paragonie/random_compat" = { - targetDir = ""; - src = composerEnv.buildZipPackage { - name = "paragonie-random_compat-996434e5492cb4c3edcb9168db6fbb1359ef965a"; - src = fetchurl { - url = "https://api.github.com/repos/paragonie/random_compat/zipball/996434e5492cb4c3edcb9168db6fbb1359ef965a"; - sha256 = "0ky7lal59dihf969r1k3pb96ql8zzdc5062jdbg69j6rj0scgkyx"; - }; - }; - }; - "php-amqplib/php-amqplib" = { - targetDir = ""; - src = composerEnv.buildZipPackage { - name = "php-amqplib-php-amqplib-cb514530ce45a6d2f636be5196010c47c3bcf6e0"; - src = fetchurl { - url = "https://api.github.com/repos/php-amqplib/php-amqplib/zipball/cb514530ce45a6d2f636be5196010c47c3bcf6e0"; - sha256 = "0mjca0m9960m8xgi22azwk8v1lgg8yznxscw10sqfzgp4wj4sfv0"; - }; - }; - }; - "phpseclib/phpseclib" = { - targetDir = ""; - src = composerEnv.buildZipPackage { - name = "phpseclib-phpseclib-cfa2013d0f68c062055180dd4328cc8b9d1f30b8"; - src = fetchurl { - url = "https://api.github.com/repos/phpseclib/phpseclib/zipball/cfa2013d0f68c062055180dd4328cc8b9d1f30b8"; - sha256 = "1wgzy4fbj565czpn9xasr8lnd9ilh1x3bsalrpx5bskvqr4zspgj"; - }; - }; - }; - "svanderburg/composer2nix" = { - targetDir = ""; - src = composerEnv.buildZipPackage { - name = "svanderburg-composer2nix-299caca4aac42d7639a42eb4dde951c010f6e91c"; - src = fetchurl { - url = "https://api.github.com/repos/svanderburg/composer2nix/zipball/299caca4aac42d7639a42eb4dde951c010f6e91c"; - sha256 = "0vb7q4za6z89azz4c5v7hgcv9gblcpk7hffl6va7q5f27fyyhwy0"; - }; - }; - }; - "svanderburg/pndp" = { - targetDir = ""; - src = composerEnv.buildZipPackage { - name = "svanderburg-pndp-bc795b341d95c24bb577e0d7a4a37fde98b1cce8"; - src = fetchurl { - url = "https://api.github.com/repos/svanderburg/pndp/zipball/bc795b341d95c24bb577e0d7a4a37fde98b1cce8"; - sha256 = "1y46wsccjwdkvs1c1bklwbp7crsg0axyr7ncdibbny1sr54xb24i"; - }; - }; - }; - }; - devPackages = {}; -in -composerEnv.buildPackage { - inherit packages devPackages noDev; - name = "nixos-ofborg-webhook"; - src = composerEnv.filterSrc ./.; - executable = false; - symlinkDependencies = false; - meta = {}; -} diff --git a/php/web/index.php b/php/web/index.php deleted file mode 100644 index e22331a0..00000000 --- a/php/web/index.php +++ /dev/null @@ -1,186 +0,0 @@ -repository)) { - throw new ExecutionFailureException('Dataset does not have a repository'); - } - - if (!isset($input->repository->full_name)) { - throw new ExecutionFailureException('Dataset repository does not have a name'); - } - - $name = strtolower($input->repository->full_name); - $eventtype = event_type(); - - $connection = retry_rabbitmq_conn(); - $channel = $connection->channel(); - - $dec = $channel->exchange_declare( - 'github-events', - 'topic', - false, // passive - true, // durable - false // auto_delete - ); - - $channel->queue_declare( - 'github-events-unknown', - false, // passive - true, // durable - false, // exclusive - false // auto-delete - ); - $channel->queue_bind( - 'github-events-unknown', - 'github-events', - 'unknown.*' - ); - - $message = new AMQPMessage(json_encode($input), - array( - 'content_type' => 'application/json', - 'delivery_mode' => AMQPMessage::DELIVERY_MODE_PERSISTENT, - )); - - $routing_key = "$eventtype.$name"; - $rec = $channel->basic_publish($message, 'github-events', $routing_key); - - echo "ok"; -} catch (DumpableException $e) { - trigger_error(print_r($e, true), E_USER_WARNING); - header("HTTP/1.1 400 Eh", true, 400); - var_dump($e); - echo ob_get_clean(); -} catch (\Exception $e) { - trigger_error(print_r($e, true), E_USER_WARNING); - header("HTTP/1.1 400 Meh", true, 400); - var_dump(get_class($e)); - echo ob_get_clean(); -} \ No newline at end of file From 671fa9306d44f2869e96d1e75a60e67d4957b784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Mon, 23 Dec 2024 20:33:15 +0100 Subject: [PATCH 02/10] Rewrite the webhook receiver in Rust --- Cargo.lock | 24 ++- config.public.json | 11 ++ flake.nix | 1 + ofborg/Cargo.toml | 3 + ofborg/src/bin/github-webhook-receiver.rs | 207 ++++++++++++++++++++++ ofborg/src/config.rs | 14 ++ ofborg/src/ghevent/common.rs | 8 + ofborg/src/ghevent/mod.rs | 2 +- 8 files changed, 267 insertions(+), 3 deletions(-) create mode 100644 ofborg/src/bin/github-webhook-receiver.rs diff --git a/Cargo.lock b/Cargo.lock index 45ee1b0b..48b94d64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -527,9 +527,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.5" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -775,6 +775,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hmac" version = "0.12.1" @@ -1269,6 +1275,8 @@ dependencies = [ "either", "fs2", "futures-util", + "hex", + "hmac", "http", "hubcaps", "hyper 0.10.16", @@ -1282,6 +1290,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "sha2", "sys-info", "tempfile", "tracing", @@ -1750,6 +1759,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.4" diff --git a/config.public.json b/config.public.json index 43167859..1b8ad162 100644 --- a/config.public.json +++ b/config.public.json @@ -1,4 +1,15 @@ { + "github_webhook_receiver": { + "listen": "[::1]:9899", + "webhook_secret_file: "/run/secrets/ofborg/github-webhook-secret", + "rabbitmq": { + "host": "localhost", + "ssl": false, + "username": "ofborg-github-webhook", + "password_file": "/run/secrets/ofborg/github-webhook-amqp-password", + "virtualhost": "ofborg" + } + }, "feedback": { "full_logs": true }, diff --git a/flake.nix b/flake.nix index b4a11d85..82cab4eb 100644 --- a/flake.nix +++ b/flake.nix @@ -122,6 +122,7 @@ test -e $out/bin/builder test -e $out/bin/github_comment_filter test -e $out/bin/github_comment_poster + test -e $out/bin/github_webhook_receiver test -e $out/bin/log_message_collector test -e $out/bin/evaluation_filter ''; diff --git a/ofborg/Cargo.toml b/ofborg/Cargo.toml index 5360f986..4843504e 100644 --- a/ofborg/Cargo.toml +++ b/ofborg/Cargo.toml @@ -34,3 +34,6 @@ tracing = "0.1.37" tracing-subscriber = { version = "0.3.16", features = ["json", "env-filter"] } uuid = { version = "1.2", features = ["v4"] } rustls-pemfile = "1.0.2" +hmac = "0.12.1" +sha2 = "0.10.8" +hex = "0.4.3" diff --git a/ofborg/src/bin/github-webhook-receiver.rs b/ofborg/src/bin/github-webhook-receiver.rs new file mode 100644 index 00000000..89170c3e --- /dev/null +++ b/ofborg/src/bin/github-webhook-receiver.rs @@ -0,0 +1,207 @@ +use std::env; +use std::error::Error; +use std::io::Read as _; +use std::sync::Arc; +#[macro_use] +extern crate hyper; + +use async_std::task; +use hmac::{Hmac, Mac}; +use hyper::header; +use hyper::{ + server::{Request, Response, Server}, + status::StatusCode, +}; +use lapin::options::BasicPublishOptions; +use lapin::{BasicProperties, Channel}; +use ofborg::ghevent::GenericWebhook; +use ofborg::{config, easyamqp, easyamqp::ChannelExt, easylapin}; +use sha2::Sha256; +use tracing::{error, info, warn}; + +header! { (XHubSignature256, "X-Hub-Signature-256") => [String] } +header! { (XGithubEvent, "X-Github-Event") => [String] } + +/// Prepares the the exchange we will write to, the queues that are bound to it +/// and binds them. +fn setup_amqp(chan: &mut Channel) -> Result<(), Box> { + chan.declare_exchange(easyamqp::ExchangeConfig { + exchange: "github-events".to_owned(), + exchange_type: easyamqp::ExchangeType::Topic, + passive: false, + durable: true, + auto_delete: false, + no_wait: false, + internal: false, + })?; + + let queue_name = String::from("build-inputs"); + chan.declare_queue(easyamqp::QueueConfig { + queue: queue_name.clone(), + passive: false, + durable: true, + exclusive: false, + auto_delete: false, + no_wait: false, + })?; + chan.bind_queue(easyamqp::BindQueueConfig { + queue: queue_name.clone(), + exchange: "github-events".to_owned(), + routing_key: Some(String::from("issue_comment.*")), + no_wait: false, + })?; + + let queue_name = String::from("github-events-unknown"); + chan.declare_queue(easyamqp::QueueConfig { + queue: queue_name.clone(), + passive: false, + durable: true, + exclusive: false, + auto_delete: false, + no_wait: false, + })?; + chan.bind_queue(easyamqp::BindQueueConfig { + queue: queue_name.clone(), + exchange: "github-events".to_owned(), + routing_key: Some(String::from("unknown.*")), + no_wait: false, + })?; + + let queue_name = String::from("mass-rebuild-check-inputs"); + chan.declare_queue(easyamqp::QueueConfig { + queue: queue_name.clone(), + passive: false, + durable: true, + exclusive: false, + auto_delete: false, + no_wait: false, + })?; + chan.bind_queue(easyamqp::BindQueueConfig { + queue: queue_name.clone(), + exchange: "github-events".to_owned(), + routing_key: Some(String::from("pull_request.nixos/nixpkgs")), + no_wait: false, + })?; + Ok(()) +} + +fn main() -> Result<(), Box> { + ofborg::setup_log(); + + let arg = env::args() + .nth(1) + .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); + let Some(cfg) = config::load(arg.as_ref()).github_webhook_receiver else { + error!("No GitHub Webhook configuration found!"); + panic!(); + }; + + let webhook_secret = std::fs::read_to_string(cfg.webhook_secret_file) + .expect("Unable to read webhook secret file"); + let webhook_secret = Arc::new(webhook_secret.trim().to_string()); + + let conn = easylapin::from_config(&cfg.rabbitmq)?; + let mut chan = task::block_on(conn.create_channel())?; + setup_amqp(&mut chan)?; + + //let events = stats::RabbitMq::from_lapin(&cfg.whoami(), task::block_on(conn.create_channel())?); + let threads = std::thread::available_parallelism() + .map(|x| x.get()) + .unwrap_or(1); + info!("Will listen on {} with {threads} threads", cfg.listen); + Server::http(cfg.listen)?.handle_threads( + move |mut req: Request, mut res: Response| { + // HTTP 405 + if req.method != hyper::Post { + *res.status_mut() = StatusCode::MethodNotAllowed; + return; + } + let hdr = req.headers.clone(); + + // Read body + let mut raw = Vec::new(); + if req.read_to_end(&mut raw).is_err() { + warn!("Failed to read body from client"); + *res.status_mut() = StatusCode::InternalServerError; + return; + } + let raw = raw.as_slice(); + + // Validate signature + { + let Some(sig) = hdr.get::() else { + *res.status_mut() = StatusCode::BadRequest; + let _ = res.send(b"Missing signature header"); + return; + }; + let mut components = sig.splitn(2, '='); + let Some(algo) = components.next() else { + *res.status_mut() = StatusCode::BadRequest; + let _ = res.send(b"Signature hash method missing"); + return; + }; + let Some(hash) = components.next() else { + *res.status_mut() = StatusCode::BadRequest; + let _ = res.send(b"Signature hash missing"); + return; + }; + let Ok(hash) = hex::decode(hash) else { + *res.status_mut() = StatusCode::BadRequest; + let _ = res.send(b"Invalid signature hash hex"); + return; + }; + + if algo != "sha256" { + *res.status_mut() = StatusCode::BadRequest; + let _ = res.send(b"Invalid signature hash method"); + return; + } + + let Ok(mut mac) = Hmac::::new_from_slice(webhook_secret.as_bytes()) else { + *res.status_mut() = StatusCode::InternalServerError; + error!("Unable to create HMAC from secret"); + return; + }; + mac.update(raw); + if mac.verify_slice(hash.as_slice()).is_err() { + *res.status_mut() = StatusCode::BadRequest; + let _ = res.send(b"Signature verification failed"); + return; + } + } + + // Parse body + let Ok(input) = serde_json::from_slice::(raw) else { + *res.status_mut() = StatusCode::BadRequest; + let _ = res.send(b"Invalid JSON"); + error!("Invalid JSON received"); + return; + }; + + // Build routing key + let Some(event_type) = hdr.get::() else { + *res.status_mut() = StatusCode::BadRequest; + let _ = res.send(b"Missing event type"); + return; + }; + let routing_key = format!("{event_type}.{}", input.repository.full_name.to_lowercase()); + + // Publish message + let _confirmation = task::block_on(async { + chan.basic_publish( + "github-events", + &routing_key, + BasicPublishOptions::default(), + raw, + BasicProperties::default() + .with_content_type("application/json".into()) + .with_delivery_mode(2), // persistent + ) + .await + }); + *res.status_mut() = StatusCode::NoContent; + }, + threads, + )?; + Ok(()) +} diff --git a/ofborg/src/config.rs b/ofborg/src/config.rs index 45223377..95058f1d 100644 --- a/ofborg/src/config.rs +++ b/ofborg/src/config.rs @@ -14,6 +14,8 @@ use tracing::{debug, error, info, warn}; #[derive(Serialize, Deserialize, Debug)] pub struct Config { + /// Configuration for the webhook receiver + pub github_webhook_receiver: Option, pub runner: RunnerConfig, pub feedback: FeedbackConfig, pub checkout: CheckoutConfig, @@ -24,6 +26,18 @@ pub struct Config { pub log_storage: Option, } +/// Configuration for the webhook receiver +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct GithubWebhookConfig { + /// Listen host/port + pub listen: String, + /// Path to the GitHub webhook secret + pub webhook_secret_file: String, + /// RabbitMQ broker to connect to + pub rabbitmq: RabbitMqConfig, +} + #[derive(Serialize, Deserialize, Debug)] pub struct FeedbackConfig { pub full_logs: bool, diff --git a/ofborg/src/ghevent/common.rs b/ofborg/src/ghevent/common.rs index 4f22abcc..b0725386 100644 --- a/ofborg/src/ghevent/common.rs +++ b/ofborg/src/ghevent/common.rs @@ -21,3 +21,11 @@ pub struct Repository { pub struct Issue { pub number: u64, } + +/// A generic webhook that we received with minimal verification, only for handling in the GitHub +/// webhook receiver. +#[derive(Serialize, Deserialize, Debug)] +pub struct GenericWebhook { + /// The repository the event originated + pub repository: Repository, +} diff --git a/ofborg/src/ghevent/mod.rs b/ofborg/src/ghevent/mod.rs index 9767c86a..24375880 100644 --- a/ofborg/src/ghevent/mod.rs +++ b/ofborg/src/ghevent/mod.rs @@ -2,7 +2,7 @@ mod common; mod issuecomment; mod pullrequestevent; -pub use self::common::{Comment, Issue, Repository, User}; +pub use self::common::{Comment, GenericWebhook, Issue, Repository, User}; pub use self::issuecomment::{IssueComment, IssueCommentAction}; pub use self::pullrequestevent::{ PullRequest, PullRequestAction, PullRequestEvent, PullRequestState, From bc9577cf93dd0531d4f29c25ce4ee7e83ab1dcfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Mon, 23 Dec 2024 20:40:06 +0100 Subject: [PATCH 03/10] Mildly update nixpkgs checkout used --- flake.lock | 8 ++++---- flake.nix | 4 ++-- ofborg/src/nix.rs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 1e6d6eed..a6a19a13 100644 --- a/flake.lock +++ b/flake.lock @@ -2,16 +2,16 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1670543317, - "narHash": "sha256-4mMR56rtxKr+Gwz399jFr4i76SQZxsLWxxyfQlPXRm0=", + "lastModified": 1734875076, + "narHash": "sha256-Pzyb+YNG5u3zP79zoi8HXYMs15Q5dfjDgwCdUI5B0nY=", "owner": "nixos", "repo": "nixpkgs", - "rev": "7a6a010c3a1d00f8470a5ca888f2f927f1860a19", + "rev": "1807c2b91223227ad5599d7067a61665c52d1295", "type": "github" }, "original": { "owner": "nixos", - "ref": "nixos-22.11", + "ref": "nixos-24.11", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 82cab4eb..a907ea3a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,6 @@ { inputs = { - nixpkgs.url = "github:nixos/nixpkgs/nixos-22.11"; + nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11"; }; outputs = @@ -75,7 +75,7 @@ src = pkgs.nix-gitignore.gitignoreSource [ ] ./.; nativeBuildInputs = with pkgs; [ - pkgconfig + pkg-config pkgs.rustPackages.clippy ]; diff --git a/ofborg/src/nix.rs b/ofborg/src/nix.rs index 15d754f9..e63bee2b 100644 --- a/ofborg/src/nix.rs +++ b/ofborg/src/nix.rs @@ -885,7 +885,7 @@ mod tests { ret, Expect::Fail, vec![ - "access to absolute path '/fake'", + "access to URI 'git+file:///fake", "is forbidden in restricted mode", ], ); From a691815902362b08a41c0c4239d37ab66d4364af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Thu, 26 Dec 2024 10:44:25 +0100 Subject: [PATCH 04/10] gh-webhook-receiver: Make debugging easier --- ofborg/src/bin/github-webhook-receiver.rs | 28 +++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/ofborg/src/bin/github-webhook-receiver.rs b/ofborg/src/bin/github-webhook-receiver.rs index 89170c3e..175aa39b 100644 --- a/ofborg/src/bin/github-webhook-receiver.rs +++ b/ofborg/src/bin/github-webhook-receiver.rs @@ -7,7 +7,8 @@ extern crate hyper; use async_std::task; use hmac::{Hmac, Mac}; -use hyper::header; +use hyper::header::ContentType; +use hyper::mime; use hyper::{ server::{Request, Response, Server}, status::StatusCode, @@ -171,12 +172,31 @@ fn main() -> Result<(), Box> { } // Parse body - let Ok(input) = serde_json::from_slice::(raw) else { + let Some(ct) = hdr.get::() else { *res.status_mut() = StatusCode::BadRequest; - let _ = res.send(b"Invalid JSON"); - error!("Invalid JSON received"); + let _ = res.send(b"No Content-Type header passed"); return; }; + if ct + != &ContentType(mime::Mime( + mime::TopLevel::Application, + mime::SubLevel::Json, + Vec::new(), + )) + { + *res.status_mut() = StatusCode::BadRequest; + let _ = res.send(b"Content-Type is not application/json. Webhook misconfigured?"); + return; + } + let input = match serde_json::from_slice::(raw) { + Ok(i) => i, + Err(e) => { + *res.status_mut() = StatusCode::BadRequest; + let _ = res.send(b"Invalid JSON"); + error!("Invalid JSON received: {e}"); + return; + } + }; // Build routing key let Some(event_type) = hdr.get::() else { From 2bb68a7f842a4c76ef07832d0e5789895ba49b84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Thu, 26 Dec 2024 10:56:12 +0100 Subject: [PATCH 05/10] Make routing key for PRs more generic --- ofborg/src/bin/evaluation-filter.rs | 2 +- ofborg/src/bin/github-webhook-receiver.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ofborg/src/bin/evaluation-filter.rs b/ofborg/src/bin/evaluation-filter.rs index 1c5c0327..97ec8ef8 100644 --- a/ofborg/src/bin/evaluation-filter.rs +++ b/ofborg/src/bin/evaluation-filter.rs @@ -52,7 +52,7 @@ fn main() -> Result<(), Box> { chan.bind_queue(easyamqp::BindQueueConfig { queue: queue_name.clone(), exchange: "github-events".to_owned(), - routing_key: Some("pull_request.nixos/nixpkgs".to_owned()), + routing_key: Some("pull_request.nixos/*".to_owned()), no_wait: false, })?; diff --git a/ofborg/src/bin/github-webhook-receiver.rs b/ofborg/src/bin/github-webhook-receiver.rs index 175aa39b..7e911126 100644 --- a/ofborg/src/bin/github-webhook-receiver.rs +++ b/ofborg/src/bin/github-webhook-receiver.rs @@ -80,7 +80,7 @@ fn setup_amqp(chan: &mut Channel) -> Result<(), Box> { chan.bind_queue(easyamqp::BindQueueConfig { queue: queue_name.clone(), exchange: "github-events".to_owned(), - routing_key: Some(String::from("pull_request.nixos/nixpkgs")), + routing_key: Some(String::from("pull_request.*")), no_wait: false, })?; Ok(()) From 725b990427b4677fd655df20275b006e45fe05e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Thu, 26 Dec 2024 11:21:50 +0100 Subject: [PATCH 06/10] evaluation-filter: Adapt new config format --- ofborg/src/bin/evaluation-filter.rs | 11 ++++++++--- ofborg/src/config.rs | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/ofborg/src/bin/evaluation-filter.rs b/ofborg/src/bin/evaluation-filter.rs index 97ec8ef8..d391f237 100644 --- a/ofborg/src/bin/evaluation-filter.rs +++ b/ofborg/src/bin/evaluation-filter.rs @@ -2,7 +2,7 @@ use std::env; use std::error::Error; use async_std::task; -use tracing::info; +use tracing::{error, info}; use ofborg::config; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; @@ -14,10 +14,15 @@ fn main() -> Result<(), Box> { let arg = env::args() .nth(1) - .expect("usage: evaluation-filter "); + .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); - let conn = easylapin::from_config(&cfg.rabbitmq)?; + let Some(filter_cfg) = config::load(arg.as_ref()).evaluation_filter else { + error!("No evaluation filter configuration found!"); + panic!(); + }; + + let conn = easylapin::from_config(&filter_cfg.rabbitmq)?; let mut chan = task::block_on(conn.create_channel())?; chan.declare_exchange(easyamqp::ExchangeConfig { diff --git a/ofborg/src/config.rs b/ofborg/src/config.rs index 95058f1d..3f19d07b 100644 --- a/ofborg/src/config.rs +++ b/ofborg/src/config.rs @@ -12,10 +12,13 @@ use hubcaps::{Credentials, Github, InstallationTokenGenerator, JWTCredentials}; use serde::de::{self, Deserialize, Deserializer}; use tracing::{debug, error, info, warn}; +/// Main ofBorg configuration #[derive(Serialize, Deserialize, Debug)] pub struct Config { /// Configuration for the webhook receiver pub github_webhook_receiver: Option, + /// Configuration for the evaluation filter + pub evaluation_filter: Option, pub runner: RunnerConfig, pub feedback: FeedbackConfig, pub checkout: CheckoutConfig, @@ -38,17 +41,31 @@ pub struct GithubWebhookConfig { pub rabbitmq: RabbitMqConfig, } +/// Configuration for the evaluation filter +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct EvaluationFilter { + /// RabbitMQ broker to connect to + pub rabbitmq: RabbitMqConfig, +} + #[derive(Serialize, Deserialize, Debug)] pub struct FeedbackConfig { pub full_logs: bool, } +/// Configures the connection to a RabbitMQ instance #[derive(Serialize, Deserialize, Debug, Clone)] pub struct RabbitMqConfig { + /// Whether or not to use SSL pub ssl: bool, + /// Hostname to conenct to pub host: String, + /// Virtual host to use (defaults to /) pub virtualhost: Option, + /// Username to connect with pub username: String, + /// File to read the user password from. Contents are automatically stripped pub password_file: PathBuf, } @@ -86,9 +103,12 @@ pub struct RunnerConfig { #[serde(default = "default_instance")] pub instance: u8, pub identity: String, + /// List of GitHub repos we feel responsible for pub repos: Option>, + /// Whether to use the `trusted_users` field or just allow everyone #[serde(default = "Default::default")] pub disable_trusted_users: bool, + /// List of users who are allowed to build on less sandboxed platforms pub trusted_users: Option>, /// If true, will create its own queue attached to the build job From 7e4e24db52a737b007113a65da654f8bb8417016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Thu, 26 Dec 2024 11:35:32 +0100 Subject: [PATCH 07/10] github-comment-filter: Adapt new config --- ofborg/src/bin/github-comment-filter.rs | 11 ++++++++--- ofborg/src/config.rs | 10 ++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ofborg/src/bin/github-comment-filter.rs b/ofborg/src/bin/github-comment-filter.rs index acee59b9..bea2bd82 100644 --- a/ofborg/src/bin/github-comment-filter.rs +++ b/ofborg/src/bin/github-comment-filter.rs @@ -2,7 +2,7 @@ use std::env; use std::error::Error; use async_std::task; -use tracing::info; +use tracing::{error, info}; use ofborg::config; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; @@ -14,10 +14,15 @@ fn main() -> Result<(), Box> { let arg = env::args() .nth(1) - .expect("usage: github-comment-filter "); + .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); - let conn = easylapin::from_config(&cfg.rabbitmq)?; + let Some(filter_cfg) = config::load(arg.as_ref()).github_comment_filter else { + error!("No comment filter configuration found!"); + panic!(); + }; + + let conn = easylapin::from_config(&filter_cfg.rabbitmq)?; let mut chan = task::block_on(conn.create_channel())?; chan.declare_exchange(easyamqp::ExchangeConfig { diff --git a/ofborg/src/config.rs b/ofborg/src/config.rs index 3f19d07b..d61fc7f8 100644 --- a/ofborg/src/config.rs +++ b/ofborg/src/config.rs @@ -19,6 +19,8 @@ pub struct Config { pub github_webhook_receiver: Option, /// Configuration for the evaluation filter pub evaluation_filter: Option, + /// Configuration for the GitHub comment filter + pub github_comment_filter: Option, pub runner: RunnerConfig, pub feedback: FeedbackConfig, pub checkout: CheckoutConfig, @@ -49,6 +51,14 @@ pub struct EvaluationFilter { pub rabbitmq: RabbitMqConfig, } +/// Configuration for the GitHub comment filter +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct GithubCommentFilter { + /// RabbitMQ broker to connect to + pub rabbitmq: RabbitMqConfig, +} + #[derive(Serialize, Deserialize, Debug)] pub struct FeedbackConfig { pub full_logs: bool, From 707510eb243761f242e58e917f7cc64e1ab880de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Thu, 26 Dec 2024 12:28:17 +0100 Subject: [PATCH 08/10] github-comment-filter: Create build-inputs-* --- ofborg/src/bin/github-comment-filter.rs | 13 +++++++++++++ ofborg/src/systems.rs | 11 +++++++++++ 2 files changed, 24 insertions(+) diff --git a/ofborg/src/bin/github-comment-filter.rs b/ofborg/src/bin/github-comment-filter.rs index bea2bd82..aee53157 100644 --- a/ofborg/src/bin/github-comment-filter.rs +++ b/ofborg/src/bin/github-comment-filter.rs @@ -2,6 +2,7 @@ use std::env; use std::error::Error; use async_std::task; +use ofborg::systems::System; use tracing::{error, info}; use ofborg::config; @@ -62,6 +63,18 @@ fn main() -> Result<(), Box> { no_wait: false, })?; + // Create build job queues + for sys in System::all_known_systems().iter().map(System::to_string) { + chan.declare_queue(easyamqp::QueueConfig { + queue: format!("build-inputs-{sys}"), + passive: false, + durable: true, + exclusive: false, + auto_delete: false, + no_wait: false, + })?; + } + let handle = easylapin::WorkerChannel(chan).consume( tasks::githubcommentfilter::GitHubCommentWorker::new(cfg.acl(), cfg.github()), easyamqp::ConsumeConfig { diff --git a/ofborg/src/systems.rs b/ofborg/src/systems.rs index c7242482..dcf54d27 100644 --- a/ofborg/src/systems.rs +++ b/ofborg/src/systems.rs @@ -6,6 +6,17 @@ pub enum System { Aarch64Darwin, } +impl System { + pub fn all_known_systems() -> [Self; 4] { + [ + Self::X8664Linux, + Self::Aarch64Linux, + Self::X8664Darwin, + Self::Aarch64Darwin, + ] + } +} + impl std::fmt::Display for System { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { From cc196c9082f38aeac211ddbdf5051bd1ed3ad560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Thu, 26 Dec 2024 12:45:04 +0100 Subject: [PATCH 09/10] github-comment-filter: Ensure build-results exchange --- ofborg/src/bin/github-comment-filter.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ofborg/src/bin/github-comment-filter.rs b/ofborg/src/bin/github-comment-filter.rs index aee53157..1ddf045c 100644 --- a/ofborg/src/bin/github-comment-filter.rs +++ b/ofborg/src/bin/github-comment-filter.rs @@ -63,6 +63,16 @@ fn main() -> Result<(), Box> { no_wait: false, })?; + chan.declare_exchange(easyamqp::ExchangeConfig { + exchange: "build-results".to_owned(), + exchange_type: easyamqp::ExchangeType::Fanout, + passive: false, + durable: true, + auto_delete: false, + no_wait: false, + internal: false, + })?; + // Create build job queues for sys in System::all_known_systems().iter().map(System::to_string) { chan.declare_queue(easyamqp::QueueConfig { From 0702f76e12df12154ce21c3ddbbd1f8f046a8ac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Thu, 26 Dec 2024 12:46:09 +0100 Subject: [PATCH 10/10] Move to GitHub OAuth auth and remove PAT --- ofborg/src/config.rs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/ofborg/src/config.rs b/ofborg/src/config.rs index d61fc7f8..2812caa6 100644 --- a/ofborg/src/config.rs +++ b/ofborg/src/config.rs @@ -26,7 +26,6 @@ pub struct Config { pub checkout: CheckoutConfig, pub nix: NixConfig, pub rabbitmq: RabbitMqConfig, - pub github: Option, pub github_app: Option, pub log_storage: Option, } @@ -88,15 +87,12 @@ pub struct NixConfig { pub initial_heap_size: Option, } -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct GithubConfig { - pub token_file: PathBuf, -} - #[derive(Serialize, Deserialize, Debug, Clone)] pub struct GithubAppConfig { pub app_id: u64, pub private_key: PathBuf, + pub oauth_client_id: String, + pub oauth_client_secret_file: PathBuf, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -162,12 +158,12 @@ impl Config { } pub fn github(&self) -> Github { - let token = std::fs::read_to_string(self.github.clone().unwrap().token_file) - .expect("Couldn't read from GitHub token file"); + let token = std::fs::read_to_string(self.github_app.clone().expect("No GitHub app configured").oauth_client_secret_file) + .expect("Couldn't read from GitHub app token"); + let token = token.trim(); Github::new( - "github.com/grahamc/ofborg", - // tls configured hyper client - Credentials::Token(token), + "github.com/ofborg/ofborg", + Credentials::Client(self.github_app.clone().expect("No GitHub app configured").oauth_client_id, token.to_owned()), ) .expect("Unable to create a github client instance") }