Merge pull request #277647 from viraptor/bump-archivebox-2

archivebox: 0.6.2 -> 0.7.2
NixOS · Jan 10, 2024 · a962df0 · a962df0
2 parents 9022b7f + db0c2f5
commit a962df0
Show file tree

Hide file tree

Showing 4 changed files with 270 additions and 9 deletions.
diff --git a/pkgs/applications/misc/archivebox/default.nix b/pkgs/applications/misc/archivebox/default.nix
@@ -1,7 +1,16 @@
 { lib
+, stdenv
 , python3
 , fetchFromGitHub
 , fetchPypi
+, curl
+, wget
+, git
+, ripgrep
+, postlight-parser
+, readability-extractor
+, chromium
+, yt-dlp
 }:
 
 let
@@ -34,38 +43,60 @@ let
           rev = "e43f383dae3a35237e42f6acfe1207a8e7e7bdf5";
           hash = "sha256-NAMa78KhAuoJfp0Cb0Codz84sRfRQ1JhSLNYRI4GBPM=";
         };
+        # possibly a real issue, but that version is not supported anymore
+        disabledTests = [ "test_should_highlight_bash_syntax_without_name" ];
       });
     };
   };
 in
 
 python.pkgs.buildPythonApplication rec {
   pname = "archivebox";
-  version = "0.6.2";
+  version = "0.7.2";
+  pyproject = true;
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "sha256-zHty7lTra6yab9d0q3EqsPG3F+lrnZL6PjQAbL1A2NY=";
+    hash = "sha256-hdBUEX2tOWN2b11w6aG3x7MP7KQTj4Rwc2w8XvABGf4=";
   };
 
+  nativeBuildInputs = with python.pkgs; [
+    pdm-backend
+  ];
+
   propagatedBuildInputs = with python.pkgs; [
-    requests
-    mypy-extensions
+    croniter
+    dateparser
     django
     django-extensions
-    dateparser
-    youtube-dl
+    ipython
+    mypy-extensions
     python-crontab
-    croniter
+    requests
     w3lib
-    ipython
+    yt-dlp
   ];
 
+  makeWrapperArgs = [
+    "--set USE_NODE True" # used through dependencies, not needed explicitly
+    "--set READABILITY_BINARY ${lib.meta.getExe readability-extractor}"
+    "--set MERCURY_BINARY ${lib.meta.getExe postlight-parser}"
+    "--set CURL_BINARY ${lib.meta.getExe curl}"
+    "--set RIPGREP_BINARY ${lib.meta.getExe ripgrep}"
+    "--set WGET_BINARY ${lib.meta.getExe wget}"
+    "--set GIT_BINARY ${lib.meta.getExe git}"
+    "--set YOUTUBEDL_BINARY ${lib.meta.getExe yt-dlp}"
+  ] ++ (if (lib.meta.availableOn stdenv.hostPlatform chromium) then [
+    "--set CHROME_BINARY ${chromium}/bin/chromium-browser"
+  ] else [
+    "--set-default USE_CHROME False"
+  ]);
+
   meta = with lib; {
     description = "Open source self-hosted web archiving";
     homepage = "https://archivebox.io";
     license = licenses.mit;
-    maintainers = with maintainers; [ siraben ];
+    maintainers = with maintainers; [ siraben viraptor ];
     platforms = platforms.unix;
   };
 }
diff --git a/pkgs/by-name/po/postlight-parser/package.json b/pkgs/by-name/po/postlight-parser/package.json
@@ -0,0 +1,165 @@
+{
+  "name": "@postlight/parser",
+  "version": "2.2.3",
+  "description": "Postlight Parser transforms web pages into clean text. Publishers and programmers use it to make the web make sense, and readers use it to read any web article comfortably.",
+  "author": "Postlight <[email protected]>",
+  "homepage": "https://reader.postlight.com",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/postlight/parser.git"
+  },
+  "bugs": {
+    "url": "https://github.com/postlight/parser/issues"
+  },
+  "keywords": [
+    "mercury",
+    "parser",
+    "reader",
+    "web",
+    "content"
+  ],
+  "files": [
+    "dist",
+    "cli.js",
+    "src/shims/"
+  ],
+  "main": "./dist/mercury.js",
+  "bin": {
+    "mercury-parser": "./cli.js",
+    "postlight-parser": "./cli.js"
+  },
+  "scripts": {
+    "lint": "eslint . --fix",
+    "lint:ci": "remark . && eslint .",
+    "lint-fix-quiet": "eslint --fix --quiet",
+    "build": "yarn lint && rollup -c && yarn test:build",
+    "build:ci": "rollup -c && yarn test:build",
+    "build:web": "yarn lint && rollup -c rollup.config.web.js && yarn test:build:web",
+    "build:esm": "yarn lint && rollup -c rollup.config.esm.js && yarn test:build:esm",
+    "build:esm:ci": "rollup -c rollup.config.esm.js && yarn test:build:esm",
+    "build:web:ci": "rollup -c rollup.config.web.js && yarn test:build:web",
+    "release": "yarn build && yarn build:web",
+    "build:generator": "rollup -c scripts/rollup.config.js",
+    "test_build": "rollup -c",
+    "test": "yarn test:node && yarn test:web",
+    "test:node": "jest --json --outputFile test-output.json",
+    "test:web": "node ./node_modules/karma/bin/karma start karma.conf.js --auto-watch",
+    "test:build": "cd ./scripts && jest check-build.test.js",
+    "test:build:web": "node ./scripts/proxy-browser-test.js",
+    "test:build:esm": "node ./scripts/proxy-browser-test.js",
+    "watch:test": "jest --watch",
+    "generate-parser": "node ./dist/generate-custom-parser.js"
+  },
+  "engines": {
+    "node": ">=10"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.0.0",
+    "@babel/plugin-transform-runtime": "^7.0.0",
+    "@babel/polyfill": "^7.0.0",
+    "@babel/preset-env": "^7.0.0",
+    "@babel/runtime": "^7.0.0",
+    "@jesses/circle-github-bot": "^2.1.0",
+    "@octokit/rest": "^16.9.0",
+    "babel-core": "^7.0.0-bridge.0",
+    "babel-eslint": "^10.0.1",
+    "babel-jest": "^23.4.2",
+    "babel-plugin-module-alias": "^1.6.0",
+    "babel-plugin-module-resolver": "^3.1.2",
+    "babelify": "^10.0.0",
+    "babelrc-rollup": "^3.0.0",
+    "brfs": "^2.0.1",
+    "brfs-babel": "^2.0.0",
+    "browserify": "^16.2.3",
+    "changelog-maker": "^2.3.0",
+    "eslint": "^5.12.0",
+    "eslint-config-airbnb": "^17.1.0",
+    "eslint-config-prettier": "^6.1.0",
+    "eslint-import-resolver-babel-module": "^2.2.1",
+    "eslint-plugin-babel": "^5.3.0",
+    "eslint-plugin-import": "^2.14.0",
+    "eslint-plugin-jsx-a11y": "^6.1.2",
+    "eslint-plugin-react": "^7.12.3",
+    "express": "^4.16.4",
+    "husky": "^3.0.0",
+    "inquirer": "^7.0.0",
+    "jasmine-core": "^2.5.2",
+    "jest": "^23.6.0",
+    "jest-cli": "^23.6.0",
+    "karma": "^6.3.16",
+    "karma-browserify": "8.1.0",
+    "karma-chrome-launcher": "^3.0.0",
+    "karma-cli": "^2.0.0",
+    "karma-jasmine": "^1.0.2",
+    "karma-mocha": "^1.3.0",
+    "karma-requirejs": "^1.1.0",
+    "lint-staged": "^8.1.0",
+    "mocha": "^6.0.0",
+    "nock": "^10.0.6",
+    "ora": "^4.0.0",
+    "prettier": "^1.15.3",
+    "remark-cli": "^7.0.0",
+    "remark-lint": "^6.0.4",
+    "remark-preset-lint-recommended": "^3.0.2",
+    "request": "^2.88.2",
+    "requirejs": "^2.3.6",
+    "rollup": "^1.1.0",
+    "rollup-plugin-babel": "^4.0.1",
+    "rollup-plugin-commonjs": "^9.2.0",
+    "rollup-plugin-node-globals": "^1.4.0",
+    "rollup-plugin-node-resolve": "^2.0.0",
+    "rollup-plugin-terser": "^6.1.0",
+    "rollup-plugin-uglify": "^6.0.1",
+    "watchify": "^3.11.1"
+  },
+  "dependencies": {
+    "@babel/runtime-corejs2": "^7.2.0",
+    "@postlight/ci-failed-test-reporter": "^1.0",
+    "browser-request": "github:postlight/browser-request#feat-add-headers-to-response",
+    "cheerio": "^0.22.0",
+    "difflib": "github:postlight/difflib.js",
+    "ellipsize": "0.1.0",
+    "iconv-lite": "0.5.0",
+    "jquery": "^3.5.0",
+    "moment": "^2.23.0",
+    "moment-parseformat": "3.0.0",
+    "moment-timezone": "0.5.37",
+    "postman-request": "^2.88.1-postman.31",
+    "string-direction": "^0.1.2",
+    "turndown": "^7.1.1",
+    "valid-url": "^1.0.9",
+    "wuzzy": "^0.1.4",
+    "yargs-parser": "^15.0.1"
+  },
+  "bundleDependencies": [
+    "jquery",
+    "moment-timezone",
+    "browser-request"
+  ],
+  "browser": {
+    "main": "./dist/mercury.web.js",
+    "cheerio": "./src/shims/cheerio-query",
+    "jquery": "./node_modules/jquery/dist/jquery.min.js",
+    "postman-request": "browser-request",
+    "iconv-lite": "./src/shims/iconv-lite",
+    "moment-timezone": "./node_modules/moment-timezone/builds/moment-timezone-with-data-2012-2022.min.js"
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "lint-staged"
+    }
+  },
+  "lint-staged": {
+    "*.js": [
+      "eslint --fix",
+      "prettier --write",
+      "git add"
+    ],
+    "*.{json,css,md}": [
+      "remark .",
+      "prettier --write",
+      "git add"
+    ]
+  }
+}
diff --git a/pkgs/by-name/po/postlight-parser/package.nix b/pkgs/by-name/po/postlight-parser/package.nix
@@ -0,0 +1,36 @@
+{ lib
+, stdenv
+, mkYarnPackage
+, fetchFromGitHub
+, fetchYarnDeps
+}:
+
+mkYarnPackage rec {
+  pname = "postlight-parser";
+  version = "2.2.3";
+
+  src = fetchFromGitHub {
+    owner = "postlight";
+    repo = "parser";
+    rev = "v${version}";
+    hash = "sha256-k6m95FHeJ+iiWSeY++1zds/bo1RtNXbnv2spaY/M+L0=";
+  };
+
+  packageJSON = ./package.json;
+
+  doDist = false;
+
+  offlineCache = fetchYarnDeps {
+    yarnLock = "${src}/yarn.lock";
+    hash = "sha256-Vs8bfkhEbPv33ew//HBeDnpQcyWveByHi1gUsdl2CNI=";
+  };
+
+  meta = with lib; {
+    changelog = "https://github.com/postlight/parser/blob/${src.rev}/CHANGELOG.md";
+    homepage = "https://reader.postlight.com";
+    description = "Extracts the bits that humans care about from any URL you give it";
+    license = licenses.mit;
+    maintainers = with maintainers; [ viraptor ];
+    mainProgram = "postlight-parser";
+  };
+}
diff --git a/pkgs/by-name/re/readability-extractor/package.nix b/pkgs/by-name/re/readability-extractor/package.nix
@@ -0,0 +1,29 @@
+{ lib
+, stdenv
+, buildNpmPackage
+, fetchFromGitHub
+}:
+
+buildNpmPackage {
+  pname = "readability-extractor";
+  version = "0.0.10";
+
+  src = fetchFromGitHub {
+    owner = "ArchiveBox";
+    repo = "readability-extractor";
+    rev = "be5c3222990d4f0459b21e74802565309bdd1d52";
+    hash = "sha256-KX9mtvwDUIV2XsH6Hgx5/W34AlM4QtZuzxp4QofPcyg=";
+  };
+
+  dontNpmBuild = true;
+
+  npmDepsHash = "sha256-bQHID9c2Ioyectx6t/GjTR/4cCyfwDfpT0aEQZoYCiU=";
+
+  meta = with lib; {
+    homepage = "https://github.com/ArchiveBox/readability-extractor";
+    description = "Javascript wrapper around Mozilla Readability for ArchiveBox to call as a oneshot CLI to extract article text";
+    license = licenses.mit;
+    maintainers = with maintainers; [ viraptor ];
+    mainProgram = "readability-extractor";
+  };
+}