From 66f683b00e77c2f8a62b7a90286a0dbdc75c300b Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Fri, 9 Oct 2020 13:08:33 -0700 Subject: [PATCH 1/9] Dynamic plugins list in the website This relies on Netlify's serverless functions capability to render plugin count (in the homepage) and plugin list (/docs/plugins) in the website! The serverless function is called "api" and it's accessible on path /.netlify/functions/api/* and has two subpaths that return JSON responses: 1. */pluginCount 2. */plugins This serverless function is built by placing a binary in /site/functions named "api" and the build instruction is in 'netlify.toml' file. Since we get plugin list by calling GitHub API, by default we run without an authentication token. However after this is merged, I will go and add a permissionless $GITHUB_ACCESS_TOKEN that I have created from my account. Unauthenticated GitHub calls are limited to 60 per IP, but since I'll add a personal access token, that should not be an issue. When debugging locally, 60 is also fine. (deploy previews for PRs from non-maintainers will not have an access token available, so they will make unauthenticated calls to GH API.) The responses from the dynamic functions are actually cached in Netlify's CDN and that helps with the rate-limiting as well. I have currently set the CDN cache duration to 1 hour for each response. Please take a look at the previews and let me know if there is anything that doesn't look good. I might delete `generate-plugin-overview` tool and update the current plugins.md in a followup PR. Signed-off-by: Ahmet Alp Balkan --- netlify.toml | 9 +- site/config.yaml | 3 + site/content/_index.md | 5 +- site/content/docs/plugins.md | 27 ++ site/content/docs/user-guide/quickstart.md | 2 +- site/content/docs/user-guide/search.md | 3 +- site/functions/go.mod | 16 ++ site/functions/go.sum | 139 ++++++++++ site/functions/server/main.go | 287 +++++++++++++++++++++ site/layouts/partials/footer.html | 72 ++++++ 10 files changed, 555 insertions(+), 8 deletions(-) create mode 100644 site/content/docs/plugins.md create mode 100644 site/functions/go.mod create mode 100644 site/functions/go.sum create mode 100644 site/functions/server/main.go diff --git a/netlify.toml b/netlify.toml index c54889f7..2c71d377 100644 --- a/netlify.toml +++ b/netlify.toml @@ -1,7 +1,8 @@ [build] base = "site/" publish = "public/" -command = "hugo" +command = "hugo && cd ./functions && go build -o api ./server" +functions = "functions/" [build.environment] HUGO_VERSION = "0.65.1" @@ -10,7 +11,9 @@ HUGO_VERSION = "0.65.1" HUGO_ENV = "production" [context.deploy-preview] -command = "hugo --buildFuture -b $DEPLOY_PRIME_URL" +command = "hugo --buildFuture -b $DEPLOY_PRIME_URL && cd ./functions && go build -o api ./server" + [context.branch-deploy] -command = "hugo --buildFuture -b $DEPLOY_PRIME_URL" +command = "hugo --buildFuture -b $DEPLOY_PRIME_URL && cd ./functions && go build -o api ./server" + diff --git a/site/config.yaml b/site/config.yaml index 2b4adfee..594f4435 100644 --- a/site/config.yaml +++ b/site/config.yaml @@ -9,3 +9,6 @@ disableKinds: markup: highlight: style: dracula + goldmark: + renderer: + unsafe: true # required for dynamic JS content editing diff --git a/site/content/_index.md b/site/content/_index.md index b7e1ef85..33cef53a 100644 --- a/site/content/_index.md +++ b/site/content/_index.md @@ -11,7 +11,8 @@ Krew helps you: - install them on your machine, - and keep the installed plugins up-to-date. -There are [over 90 kubectl plugins][list] currently distributed on Krew. +There are [over ... kubectl plugins][list] +currently distributed on Krew. Krew works across all major platforms, like macOS, Linux and Windows. @@ -20,4 +21,4 @@ plugins on multiple platforms easily and makes them discoverable through a centralized plugin repository with Krew. [kpl]: https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/ -[list]: http://sigs.k8s.io/krew-index/plugins.md +[list]: {{< relref "docs/plugins.md" >}} diff --git a/site/content/docs/plugins.md b/site/content/docs/plugins.md new file mode 100644 index 00000000..a3118412 --- /dev/null +++ b/site/content/docs/plugins.md @@ -0,0 +1,27 @@ +--- +title: Kubectl plugins available +slug: plugins +--- + +Below you will find the list of kubectl plugins distributed on the centralized +[krew-index](https://sigs.k8s.io/krew-index). To install these plugins on +your machine: + +1. [Install Krew]({{< relref "user-guide/setup/install.md" >}}) +2. Run `kubectl krew install ` to install a plugin via Krew. + + + + + + + + + + + + + + + +
NameDescriptionRepository
Loading...
diff --git a/site/content/docs/user-guide/quickstart.md b/site/content/docs/user-guide/quickstart.md index 4c3f5716..25212c3e 100644 --- a/site/content/docs/user-guide/quickstart.md +++ b/site/content/docs/user-guide/quickstart.md @@ -57,4 +57,4 @@ auth-proxy Authentication proxy to a pod or service This is pretty much all you need to know as a user to use Krew. [kpl]: https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/ -[list]: https://github.com/kubernetes-sigs/krew-index/blob/master/plugins.md +[list]: {{< relref "docs/plugins.md" >}} diff --git a/site/content/docs/user-guide/search.md b/site/content/docs/user-guide/search.md index 4416894c..a43d0c32 100644 --- a/site/content/docs/user-guide/search.md +++ b/site/content/docs/user-guide/search.md @@ -59,5 +59,4 @@ DESCRIPTION: ...{{}} ``` - -[list]: https://github.com/kubernetes-sigs/krew-index/blob/master/plugins.md +[list]: {{< relref "docs/plugins.md" >}} diff --git a/site/functions/go.mod b/site/functions/go.mod new file mode 100644 index 00000000..be110d79 --- /dev/null +++ b/site/functions/go.mod @@ -0,0 +1,16 @@ +module sigs.k8s.io/krew/site/functions + +go 1.15 + +require ( + github.com/apex/gateway v1.1.1 + github.com/aws/aws-lambda-go v1.19.1 + github.com/google/go-github/v32 v32.1.0 + github.com/pkg/errors v0.9.1 // indirect + golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be + gopkg.in/yaml.v2 v2.2.8 + sigs.k8s.io/krew v0.4.0 + sigs.k8s.io/yaml v1.2.0 +) + +// replace sigs.k8s.io/krew => ../../ diff --git a/site/functions/go.sum b/site/functions/go.sum new file mode 100644 index 00000000..a399b62f --- /dev/null +++ b/site/functions/go.sum @@ -0,0 +1,139 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/apex/gateway v1.1.1 h1:dPE3y2LQ/fSJuZikCOvekqXLyn/Wrbgt10MSECobH/Q= +github.com/apex/gateway v1.1.1/go.mod h1:x7iPY22zu9D8sfrynawEwh1wZEO/kQTRaOM5ye02tWU= +github.com/aws/aws-lambda-go v1.19.1 h1:5iUHbIZ2sG6Yq/J1IN3sWm3+vAB1CWwhI21NffLNuNI= +github.com/aws/aws-lambda-go v1.19.1/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= +github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sahilm/fuzzy v0.0.5/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.0.0-20190717022731-0bb8574e0887 h1:JVVkMN2P4a3MNzTkjgCCRwBDAGAENrCsZGLoLtQ5jvI= +k8s.io/apimachinery v0.0.0-20190717022731-0bb8574e0887/go.mod h1:sBJWIJZfxLhp7mRsRyuAE/NfKTr3kXGR1iaqg8O0gJo= +k8s.io/client-go v7.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= +sigs.k8s.io/krew v0.4.0 h1:Y9qeOcShVUKD0IAAhOwKuwivWkcTfxljZsdlJS8ATCQ= +sigs.k8s.io/krew v0.4.0/go.mod h1:e2cG2GilCwX2JjmeVblZacw7aUyHzIlL27uclbwJY98= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/site/functions/server/main.go b/site/functions/server/main.go new file mode 100644 index 00000000..dee08bca --- /dev/null +++ b/site/functions/server/main.go @@ -0,0 +1,287 @@ +// Copyright 2020 The Kubernetes Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "net/http/httputil" + "net/url" + "os" + "regexp" + "strings" + "sync" + "time" + + "github.com/apex/gateway" + "github.com/google/go-github/v32/github" + "github.com/pkg/errors" + "golang.org/x/oauth2" + "sigs.k8s.io/krew/pkg/constants" + krew "sigs.k8s.io/krew/pkg/index" + "sigs.k8s.io/yaml" +) + +const ( + orgName = "kubernetes-sigs" + repoName = "krew-index" + pluginsDir = "plugins" + + urlFetchBatchSize = 40 + cacheSeconds = 60 * 60 +) + +var ( + githubRepoPattern = regexp.MustCompile(`.*github\.com/([^/]+/[^/#]+)`) +) + +type PluginCountResponse struct { + Data struct { + Count int `json:"count"` + } `json:"data"` + Error ErrorResponse `json:"error,omitempty"` +} + +type pluginInfo struct { + Name string `json:"name,omitempty"` + Homepage string `json:"homepage,omitempty"` + ShortDescription string `json:"short_description,omitempty"` + GithubRepo string `json:"github_repo,omitempty"` +} + +type ErrorResponse struct { + Message string `json:"message,omitempty"` +} + +type PluginsResponse struct { + Data struct { + Plugins []pluginInfo `json:"plugins,omitempty"` + } `json:"data,omitempty"` + Error ErrorResponse `json:"error"` +} + +func githubClient(ctx context.Context) *github.Client { + var hc *http.Client + + // if not configured, you should configure a GITHUB_ACCESS_TOKEN + // variable on Netlify dashboard for the site. You can create a + // permission-less "personal access token" on GitHub account settings. + if v := os.Getenv("GITHUB_ACCESS_TOKEN"); v != "" { + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: v}) + hc = oauth2.NewClient(ctx, ts) + } + return github.NewClient(hc) +} + +func pluginCountHandler(w http.ResponseWriter, req *http.Request) { + _, dir, resp, err := githubClient(req.Context()). + Repositories.GetContents(req.Context(), orgName, repoName, pluginsDir, &github.RepositoryContentGetOptions{}) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + writeJSON(w, PluginCountResponse{Error: ErrorResponse{Message: fmt.Sprintf("error retrieving repo contents: %v", err)}}) + return + } + yamls := filterYAMLs(dir) + count := len(yamls) + log.Printf("github response=%s count=%d rate: limit=%d remaining=%d", + resp.Status, count, resp.Rate.Limit, resp.Rate.Remaining) + + var out PluginCountResponse + out.Data.Count = count + + w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", cacheSeconds)) + writeJSON(w, out) +} + +func writeJSON(w io.Writer, v interface{}) { + e := json.NewEncoder(w) + e.SetIndent("", " ") + if err := e.Encode(v); err != nil { + log.Printf("json write error: %v", err) + } +} + +func filterYAMLs(entries []*github.RepositoryContent) []*github.RepositoryContent { + var out []*github.RepositoryContent + for _, v := range entries { + if v == nil { + continue + } + if v.GetType() == "file" && strings.HasSuffix(v.GetName(), constants.ManifestExtension) { + out = append(out, v) + } + } + return out +} + +func loggingHandler(f http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + start := time.Now() + log.Printf("[req] > method=%s path=%s", req.Method, req.URL) + defer func() { + log.Printf("[resp] < method=%s path=%s took=%v", req.Method, req.URL, time.Since(start)) + }() + f.ServeHTTP(w, req) + }) +} + +func pluginsHandler(w http.ResponseWriter, req *http.Request) { + _, dir, resp, err := githubClient(req.Context()). + Repositories.GetContents(req.Context(), orgName, repoName, pluginsDir, &github.RepositoryContentGetOptions{}) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + writeJSON(w, PluginsResponse{Error: ErrorResponse{Message: fmt.Sprintf("error retrieving repo contents: %v", err)}}) + return + } + log.Printf("github response=%s rate: limit=%d remaining=%d", + resp.Status, resp.Rate.Limit, resp.Rate.Remaining) + var out PluginsResponse + + plugins, err := fetchPlugins(filterYAMLs(dir)) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + writeJSON(w, PluginsResponse{Error: ErrorResponse{Message: fmt.Sprintf("failed to fetch plugins: %v", err)}}) + return + } + + for _, v := range plugins { + pi := pluginInfo{ + Name: v.Name, + Homepage: v.Spec.Homepage, + ShortDescription: v.Spec.ShortDescription, + GithubRepo: findRepo(v.Spec.Homepage), + } + out.Data.Plugins = append(out.Data.Plugins, pi) + } + + w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", cacheSeconds)) + writeJSON(w, out) +} + +func fetchPlugins(entries []*github.RepositoryContent) ([]*krew.Plugin, error) { + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var ( + mu sync.Mutex + out []*krew.Plugin + retErr error + ) + + queue := make(chan string) + var wg sync.WaitGroup + + for i := 0; i < urlFetchBatchSize; i++ { + wg.Add(1) + go func(j int) { + defer wg.Done() + for { + select { + case <-ctx.Done(): + return + case url, ok := <-queue: + if !ok { + return + } + p, err := readPlugin(url) + if err != nil { + retErr = err + cancel() + return + } + mu.Lock() + out = append(out, p) + mu.Unlock() + } + } + }(i) + } + + for _, v := range entries { + url := v.GetDownloadURL() + select { + case <-ctx.Done(): + break + case queue <- url: + } + } + + close(queue) + wg.Wait() + + return out, retErr +} + +func readPlugin(url string) (*krew.Plugin, error) { + resp, err := http.Get(url) + if err != nil { + return nil, errors.Wrapf(err,"failed to get %s", url) + } + if resp.Body != nil { + defer resp.Body.Close() + } + var v krew.Plugin + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrapf(err,"failed to read file %s: %w", url) + } + + if err = yaml.Unmarshal(b, &v); err != nil { + return nil, errors.Wrapf(err,"failed to parse plugin manifest for %s", url) + } + return &v, nil +} + +func findRepo(homePage string) string { + if matches := githubRepoPattern.FindStringSubmatch(homePage); matches != nil { + return matches[1] + } + + knownHomePages := map[string]string{ + `https://krew.sigs.k8s.io/`: "kubernetes-sigs/krew", + `https://sigs.k8s.io/krew`: "kubernetes-sigs/krew", + `https://kubernetes.github.io/ingress-nginx/kubectl-plugin/`: "kubernetes/ingress-nginx", + `https://kudo.dev/`: "kudobuilder/kudo", + `https://kubevirt.io`: "kubevirt/kubectl-virt-plugin", + `https://popeyecli.io`: "derailed/popeye", + `https://soluble-ai.github.io/kubetap/`: "soluble-ai/kubetap", + } + return knownHomePages[homePage] +} + +func main() { + port := flag.Int("port", -1, "specify a port to use http rather than AWS Lambda") + flag.Parse() + + mux := http.NewServeMux() + mux.HandleFunc("/.netlify/functions/api/pluginCount", pluginCountHandler) + mux.HandleFunc("/.netlify/functions/api/plugins", pluginsHandler) + // To debug locally, you can run this server with -port=:8080 and run "hugo serve" and uncomment this: + mux.Handle("/", httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "http", Host: "localhost:1313"})) + + handler := loggingHandler(mux) + if *port == -1 { + log.Fatal(gateway.ListenAndServe("n/a", handler)) + } + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), handler)) +} diff --git a/site/layouts/partials/footer.html b/site/layouts/partials/footer.html index fda5921f..8018f7ef 100644 --- a/site/layouts/partials/footer.html +++ b/site/layouts/partials/footer.html @@ -44,5 +44,77 @@ {{ end }} + + + From 59a65d98ef571fe43b1391c26ac38c28feb6e147 Mon Sep 17 00:00:00 2001 From: Cornelius Weig <22861411+corneliusweig@users.noreply.github.com> Date: Fri, 16 Oct 2020 10:59:53 +0200 Subject: [PATCH 2/9] Apply review comments --- site/content/_index.md | 2 +- site/functions/go.mod | 2 +- site/functions/server/main.go | 41 +++++++++++++++++------------------ 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/site/content/_index.md b/site/content/_index.md index 33cef53a..2d8d73a8 100644 --- a/site/content/_index.md +++ b/site/content/_index.md @@ -11,7 +11,7 @@ Krew helps you: - install them on your machine, - and keep the installed plugins up-to-date. -There are [over ... kubectl plugins][list] +There are [... kubectl plugins][list] currently distributed on Krew. Krew works across all major platforms, like macOS, Linux and Windows. diff --git a/site/functions/go.mod b/site/functions/go.mod index be110d79..ea080052 100644 --- a/site/functions/go.mod +++ b/site/functions/go.mod @@ -6,7 +6,7 @@ require ( github.com/apex/gateway v1.1.1 github.com/aws/aws-lambda-go v1.19.1 github.com/google/go-github/v32 v32.1.0 - github.com/pkg/errors v0.9.1 // indirect + github.com/pkg/errors v0.9.1 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be gopkg.in/yaml.v2 v2.2.8 sigs.k8s.io/krew v0.4.0 diff --git a/site/functions/server/main.go b/site/functions/server/main.go index dee08bca..c1fbda71 100644 --- a/site/functions/server/main.go +++ b/site/functions/server/main.go @@ -113,7 +113,6 @@ func pluginCountHandler(w http.ResponseWriter, req *http.Request) { func writeJSON(w io.Writer, v interface{}) { e := json.NewEncoder(w) - e.SetIndent("", " ") if err := e.Encode(v); err != nil { log.Printf("json write error: %v", err) } @@ -144,8 +143,9 @@ func loggingHandler(f http.Handler) http.Handler { } func pluginsHandler(w http.ResponseWriter, req *http.Request) { - _, dir, resp, err := githubClient(req.Context()). - Repositories.GetContents(req.Context(), orgName, repoName, pluginsDir, &github.RepositoryContentGetOptions{}) + ctx := req.Context() + _, dir, resp, err := githubClient(ctx). + Repositories.GetContents(ctx, orgName, repoName, pluginsDir, &github.RepositoryContentGetOptions{}) if err != nil { w.WriteHeader(http.StatusInternalServerError) writeJSON(w, PluginsResponse{Error: ErrorResponse{Message: fmt.Sprintf("error retrieving repo contents: %v", err)}}) @@ -155,7 +155,7 @@ func pluginsHandler(w http.ResponseWriter, req *http.Request) { resp.Status, resp.Rate.Limit, resp.Rate.Remaining) var out PluginsResponse - plugins, err := fetchPlugins(filterYAMLs(dir)) + plugins, err := fetchPlugins(ctx, filterYAMLs(dir)) if err != nil { w.WriteHeader(http.StatusInternalServerError) writeJSON(w, PluginsResponse{Error: ErrorResponse{Message: fmt.Sprintf("failed to fetch plugins: %v", err)}}) @@ -176,8 +176,7 @@ func pluginsHandler(w http.ResponseWriter, req *http.Request) { writeJSON(w, out) } -func fetchPlugins(entries []*github.RepositoryContent) ([]*krew.Plugin, error) { - ctx := context.Background() +func fetchPlugins(ctx context.Context, entries []*github.RepositoryContent) ([]*krew.Plugin, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -192,7 +191,7 @@ func fetchPlugins(entries []*github.RepositoryContent) ([]*krew.Plugin, error) { for i := 0; i < urlFetchBatchSize; i++ { wg.Add(1) - go func(j int) { + go func() { defer wg.Done() for { select { @@ -213,7 +212,7 @@ func fetchPlugins(entries []*github.RepositoryContent) ([]*krew.Plugin, error) { mu.Unlock() } } - }(i) + }() } for _, v := range entries { @@ -234,20 +233,18 @@ func fetchPlugins(entries []*github.RepositoryContent) ([]*krew.Plugin, error) { func readPlugin(url string) (*krew.Plugin, error) { resp, err := http.Get(url) if err != nil { - return nil, errors.Wrapf(err,"failed to get %s", url) - } - if resp.Body != nil { - defer resp.Body.Close() + return nil, errors.Wrapf(err, "failed to get %s", url) } - var v krew.Plugin + defer resp.Body.Close() b, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, errors.Wrapf(err,"failed to read file %s: %w", url) + return nil, errors.Wrapf(err, "failed to read file %s: %w", url) } + var v krew.Plugin if err = yaml.Unmarshal(b, &v); err != nil { - return nil, errors.Wrapf(err,"failed to parse plugin manifest for %s", url) + return nil, errors.Wrapf(err, "failed to parse plugin manifest for %s", url) } return &v, nil } @@ -270,18 +267,20 @@ func findRepo(homePage string) string { } func main() { - port := flag.Int("port", -1, "specify a port to use http rather than AWS Lambda") + port := flag.Int("port", -1, `To debug locally set --port=8080 and run "hugo serve"`) flag.Parse() + local := *port != -1 mux := http.NewServeMux() mux.HandleFunc("/.netlify/functions/api/pluginCount", pluginCountHandler) mux.HandleFunc("/.netlify/functions/api/plugins", pluginsHandler) - // To debug locally, you can run this server with -port=:8080 and run "hugo serve" and uncomment this: - mux.Handle("/", httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "http", Host: "localhost:1313"})) + if local { + mux.Handle("/", httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "http", Host: "localhost:1313"})) + } handler := loggingHandler(mux) - if *port == -1 { - log.Fatal(gateway.ListenAndServe("n/a", handler)) + if local { + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), handler)) } - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), handler)) + log.Fatal(gateway.ListenAndServe("n/a", handler)) } From bf10f6513b90e92695ecbbf95b3ea9f8784305e0 Mon Sep 17 00:00:00 2001 From: Cornelius Weig <22861411+corneliusweig@users.noreply.github.com> Date: Fri, 16 Oct 2020 11:03:31 +0200 Subject: [PATCH 3/9] Use lit-html and async/await style for better readability --- site/content/_index.md | 2 +- site/layouts/partials/footer.html | 109 ++++++++++++------------------ 2 files changed, 46 insertions(+), 65 deletions(-) diff --git a/site/content/_index.md b/site/content/_index.md index 2d8d73a8..528e6883 100644 --- a/site/content/_index.md +++ b/site/content/_index.md @@ -11,7 +11,7 @@ Krew helps you: - install them on your machine, - and keep the installed plugins up-to-date. -There are [... kubectl plugins][list] +There are [⌛ kubectl plugins][list] currently distributed on Krew. Krew works across all major platforms, like macOS, Linux and Windows. diff --git a/site/layouts/partials/footer.html b/site/layouts/partials/footer.html index 8018f7ef..d3a66868 100644 --- a/site/layouts/partials/footer.html +++ b/site/layouts/partials/footer.html @@ -45,76 +45,57 @@ {{ end }} - + plugins.sort((a,b) => a.name.localeCompare(b.name)); + render(tableRows(plugins), pluginTableElem); + } catch (e) { + render(error(e), pluginTableElem); + } + }); + From 871f617eaa83e5eb76002984e89183f5200588bc Mon Sep 17 00:00:00 2001 From: Cornelius Weig <22861411+corneliusweig@users.noreply.github.com> Date: Fri, 16 Oct 2020 12:09:48 +0200 Subject: [PATCH 4/9] Simplify work sharing with errgroup --- site/functions/go.mod | 1 + site/functions/go.sum | 1 + site/functions/server/main.go | 53 +++++++++++++---------------------- 3 files changed, 21 insertions(+), 34 deletions(-) diff --git a/site/functions/go.mod b/site/functions/go.mod index ea080052..6042be03 100644 --- a/site/functions/go.mod +++ b/site/functions/go.mod @@ -8,6 +8,7 @@ require ( github.com/google/go-github/v32 v32.1.0 github.com/pkg/errors v0.9.1 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be + golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 gopkg.in/yaml.v2 v2.2.8 sigs.k8s.io/krew v0.4.0 sigs.k8s.io/yaml v1.2.0 diff --git a/site/functions/go.sum b/site/functions/go.sum index a399b62f..bb6a60e3 100644 --- a/site/functions/go.sum +++ b/site/functions/go.sum @@ -94,6 +94,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/site/functions/server/main.go b/site/functions/server/main.go index c1fbda71..511e88cc 100644 --- a/site/functions/server/main.go +++ b/site/functions/server/main.go @@ -35,6 +35,7 @@ import ( "github.com/google/go-github/v32/github" "github.com/pkg/errors" "golang.org/x/oauth2" + "golang.org/x/sync/errgroup" "sigs.k8s.io/krew/pkg/constants" krew "sigs.k8s.io/krew/pkg/index" "sigs.k8s.io/yaml" @@ -45,8 +46,8 @@ const ( repoName = "krew-index" pluginsDir = "plugins" - urlFetchBatchSize = 40 - cacheSeconds = 60 * 60 + pluginFetchWorkers = 40 + cacheSeconds = 60 * 60 ) var ( @@ -177,42 +178,27 @@ func pluginsHandler(w http.ResponseWriter, req *http.Request) { } func fetchPlugins(ctx context.Context, entries []*github.RepositoryContent) ([]*krew.Plugin, error) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - var ( - mu sync.Mutex - out []*krew.Plugin - retErr error + mu sync.Mutex + out []*krew.Plugin ) queue := make(chan string) - var wg sync.WaitGroup - - for i := 0; i < urlFetchBatchSize; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for { - select { - case <-ctx.Done(): - return - case url, ok := <-queue: - if !ok { - return - } - p, err := readPlugin(url) - if err != nil { - retErr = err - cancel() - return - } - mu.Lock() - out = append(out, p) - mu.Unlock() + g, ctx := errgroup.WithContext(ctx) + + for i := 0; i < pluginFetchWorkers; i++ { + g.Go(func() error { + for url := range queue { + p, err := readPlugin(url) + if err != nil { + return err } + mu.Lock() + out = append(out, p) + mu.Unlock() } - }() + return nil + }) } for _, v := range entries { @@ -225,9 +211,8 @@ func fetchPlugins(ctx context.Context, entries []*github.RepositoryContent) ([]* } close(queue) - wg.Wait() - return out, retErr + return out, g.Wait() } func readPlugin(url string) (*krew.Plugin, error) { From 6796c894df68403c20e6bf664d95732f5b345df4 Mon Sep 17 00:00:00 2001 From: Cornelius Weig <22861411+corneliusweig@users.noreply.github.com> Date: Sat, 17 Oct 2020 23:08:50 +0200 Subject: [PATCH 5/9] Satisfy impi constraints --- site/functions/server/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/site/functions/server/main.go b/site/functions/server/main.go index 511e88cc..6f254c01 100644 --- a/site/functions/server/main.go +++ b/site/functions/server/main.go @@ -36,6 +36,7 @@ import ( "github.com/pkg/errors" "golang.org/x/oauth2" "golang.org/x/sync/errgroup" + "sigs.k8s.io/krew/pkg/constants" krew "sigs.k8s.io/krew/pkg/index" "sigs.k8s.io/yaml" From 82c1f86b62e66dabe455779487abdce6d82661a7 Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Sat, 17 Oct 2020 21:20:44 -0700 Subject: [PATCH 6/9] fix import Signed-off-by: Ahmet Alp Balkan --- site/functions/server/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/functions/server/main.go b/site/functions/server/main.go index 6f254c01..aef0ee3e 100644 --- a/site/functions/server/main.go +++ b/site/functions/server/main.go @@ -36,10 +36,10 @@ import ( "github.com/pkg/errors" "golang.org/x/oauth2" "golang.org/x/sync/errgroup" + "sigs.k8s.io/yaml" "sigs.k8s.io/krew/pkg/constants" krew "sigs.k8s.io/krew/pkg/index" - "sigs.k8s.io/yaml" ) const ( From 9100f2e581f5cb00278f3d502bdc1396615f0916 Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Mon, 19 Oct 2020 15:12:53 -0700 Subject: [PATCH 7/9] add setup/debugging documentation Signed-off-by: Ahmet Alp Balkan --- site/functions/README.md | 41 +++++++++++++++++++++++++++++++++++ site/functions/server/main.go | 2 +- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 site/functions/README.md diff --git a/site/functions/README.md b/site/functions/README.md new file mode 100644 index 00000000..0de88019 --- /dev/null +++ b/site/functions/README.md @@ -0,0 +1,41 @@ +# Dynamic functions on Krew documentation + +Krew site makes use of Netlify Functions (lambdas) to fetch plugin list +dynamically from `krew-index` repository using GitHub API. + +## Set up on Netlify + +Functions require a one-time set up in the Netlify console. + +Regarding **GitHub API rate limits**: + +- In production, make **sure to set a `GITHUB_ACCESS_TOKEN` environment + variable** with a permissionless "personal access token" to elevate the rate + limits for our functions. + + Dynamic responses from the functions are cached on + Netlify’s CDN for a long time, so this is not a huge problem and a single + token is very likely to suffice a long period of time. + +- During local development, you can hit the GitHub API rate limit as well. + You should set the `GITHUB_ACCESS_TOKEN` environment variable as needed. + +## Local development + +Start `hugo` local iteration server on port 1313: + +``` +cd ./site +hugo serve +``` + +In another terminal window, build and start the functions server on port 8080: + +``` +cd ./functions +go run ./server -port=8080 +``` + +Now, you can reach the website at http://localhost:8080 and the functions at +paths defined in the code e.g. +http://localhost:8080/.netlify/functions/api/plugins. diff --git a/site/functions/server/main.go b/site/functions/server/main.go index aef0ee3e..1bcf552a 100644 --- a/site/functions/server/main.go +++ b/site/functions/server/main.go @@ -253,7 +253,7 @@ func findRepo(homePage string) string { } func main() { - port := flag.Int("port", -1, `To debug locally set --port=8080 and run "hugo serve"`) + port := flag.Int("port", -1, `"to debug locally, set a port number"`) flag.Parse() local := *port != -1 From 543f132e78a46d0df405b842806a3457b98d946f Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Wed, 21 Oct 2020 13:49:54 -0700 Subject: [PATCH 8/9] move from /docs/plugins to /plugins Signed-off-by: Ahmet Alp Balkan --- site/content/_index.md | 2 +- site/content/docs/user-guide/quickstart.md | 2 +- site/content/docs/user-guide/search.md | 2 +- site/content/{docs => }/plugins.md | 2 +- site/layouts/{docs => _default}/single.html | 0 5 files changed, 4 insertions(+), 4 deletions(-) rename site/content/{docs => }/plugins.md (89%) rename site/layouts/{docs => _default}/single.html (100%) diff --git a/site/content/_index.md b/site/content/_index.md index 528e6883..e3655c10 100644 --- a/site/content/_index.md +++ b/site/content/_index.md @@ -21,4 +21,4 @@ plugins on multiple platforms easily and makes them discoverable through a centralized plugin repository with Krew. [kpl]: https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/ -[list]: {{< relref "docs/plugins.md" >}} +[list]: {{< relref "plugins.md" >}} diff --git a/site/content/docs/user-guide/quickstart.md b/site/content/docs/user-guide/quickstart.md index 25212c3e..5720c990 100644 --- a/site/content/docs/user-guide/quickstart.md +++ b/site/content/docs/user-guide/quickstart.md @@ -57,4 +57,4 @@ auth-proxy Authentication proxy to a pod or service This is pretty much all you need to know as a user to use Krew. [kpl]: https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/ -[list]: {{< relref "docs/plugins.md" >}} +[list]: {{< relref "plugins.md" >}} diff --git a/site/content/docs/user-guide/search.md b/site/content/docs/user-guide/search.md index a43d0c32..2e99d8be 100644 --- a/site/content/docs/user-guide/search.md +++ b/site/content/docs/user-guide/search.md @@ -59,4 +59,4 @@ DESCRIPTION: ...{{}} ``` -[list]: {{< relref "docs/plugins.md" >}} +[list]: {{< relref "plugins.md" >}} diff --git a/site/content/docs/plugins.md b/site/content/plugins.md similarity index 89% rename from site/content/docs/plugins.md rename to site/content/plugins.md index a3118412..7b452202 100644 --- a/site/content/docs/plugins.md +++ b/site/content/plugins.md @@ -7,7 +7,7 @@ Below you will find the list of kubectl plugins distributed on the centralized [krew-index](https://sigs.k8s.io/krew-index). To install these plugins on your machine: -1. [Install Krew]({{< relref "user-guide/setup/install.md" >}}) +1. [Install Krew]({{< relref "docs/user-guide/setup/install.md" >}}) 2. Run `kubectl krew install ` to install a plugin via Krew. diff --git a/site/layouts/docs/single.html b/site/layouts/_default/single.html similarity index 100% rename from site/layouts/docs/single.html rename to site/layouts/_default/single.html From fb0b1884534160c02c980c43fd84234ca1461da8 Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Wed, 21 Oct 2020 13:51:17 -0700 Subject: [PATCH 9/9] add /plugins to navbar Signed-off-by: Ahmet Alp Balkan --- site/layouts/partials/navbar.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/site/layouts/partials/navbar.html b/site/layouts/partials/navbar.html index a3115b9a..4028a4f3 100644 --- a/site/layouts/partials/navbar.html +++ b/site/layouts/partials/navbar.html @@ -16,6 +16,7 @@ {{ $isQuickstart := (eq .Slug "quickstart") }} {{ $isDocs := (and (eq .Section "docs") (ne .Slug "quickstart")) }} + {{ $isPlugins := (and (not .Section) (eq .Slug "plugins"))}} Quickstart @@ -25,6 +26,10 @@ Documentation {{if $isDocs}}(current){{end}} + + Plugins + {{if $isPlugins}}(current){{end}} + GitHub