diff --git a/README.md b/README.md index 184cd0ae..0eee0d66 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,21 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g resources, err := ParseKubernetesYaml(values) ``` +8. To get resources from Git repositories + ```go + // Parse the repo url + gitUrl, err := util.ParseGitUrl(url) + + // Clone the repo to a destination dir + err = util.CloneGitRepo(gitUrl, destDir) + ``` + If repository is private, set the correct environment variables + ```shell + # credentials for private repositories + export GITHUB_TOKEN= + export GITLAB_TOKEN= + export BITBUCKET_TOKEN= + ``` ## Projects using devfile/library diff --git a/go.mod b/go.mod index 92530e38..6c5103eb 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/devfile/registry-support/registry-library v0.0.0-20221018213054-47b3ffaeadba github.com/fatih/color v1.7.0 github.com/fsnotify/fsnotify v1.4.9 - github.com/go-git/go-git/v5 v5.4.2 github.com/gobwas/glob v0.2.3 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.6 diff --git a/go.sum b/go.sum index 276feacd..eb341251 100644 --- a/go.sum +++ b/go.sum @@ -86,7 +86,6 @@ github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:m github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -94,7 +93,6 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -102,14 +100,12 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= @@ -342,7 +338,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -361,7 +356,6 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= @@ -372,16 +366,11 @@ github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXt github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= -github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -578,7 +567,6 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= @@ -598,7 +586,6 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -635,7 +622,6 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= @@ -657,7 +643,6 @@ github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WT github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -816,7 +801,6 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -891,7 +875,6 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= -github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1416,7 +1399,6 @@ gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76 gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index 42c17e5c..bbd3f99d 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -1,5 +1,5 @@ // -// Copyright 2022 Red Hat, Inc. +// Copyright 2022-2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/util/git.go b/pkg/util/git.go index bbf7be8d..62070285 100644 --- a/pkg/util/git.go +++ b/pkg/util/git.go @@ -1,5 +1,5 @@ // -// Copyright 2022 Red Hat, Inc. +// Copyright 2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,12 +17,9 @@ package util import ( "fmt" - gitpkg "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - githttp "github.com/go-git/go-git/v5/plumbing/transport/http" - "net/http" "net/url" "os" + "os/exec" "path/filepath" "strings" ) @@ -39,27 +36,18 @@ const ( ) type GitUrl struct { - Protocol string - Host string - Owner string - Repo string - Branch string - Path string - token string - IsFile bool + Protocol string // URL scheme + Host string // URL domain name + Owner string // name of the repo owner + Repo string // name of the repo + Branch string // branch name + Path string // path to a directory or file in the repo + token string // used for authenticating a private repo + IsFile bool // defines if the URL points to a file in the repo } // ParseGitUrl extracts information from a GitHub, GitLab, or Bitbucket url -// A client is used to check whether the url is private or public, and sets -// the providers personal access token from the environment variable func ParseGitUrl(fullUrl string) (GitUrl, error) { - var c = http.Client{ - Timeout: HTTPRequestResponseTimeout, - } - return parseGitUrlWithClient(fullUrl, c) -} - -func parseGitUrlWithClient(fullUrl string, c http.Client) (GitUrl, error) { var g GitUrl err := ValidateURL(fullUrl) @@ -77,11 +65,11 @@ func parseGitUrlWithClient(fullUrl string, c http.Client) (GitUrl, error) { } if parsedUrl.Host == RawGitHubHost || parsedUrl.Host == GitHubHost { - g, err = parseGitHubUrl(g, parsedUrl, c) + err = g.parseGitHubUrl(parsedUrl) } else if parsedUrl.Host == GitLabHost { - g, err = parseGitLabUrl(g, parsedUrl, c) + err = g.parseGitLabUrl(parsedUrl) } else if parsedUrl.Host == BitbucketHost { - g, err = parseBitbucketUrl(g, parsedUrl, c) + err = g.parseBitbucketUrl(parsedUrl) } else { err = fmt.Errorf("url host should be a valid GitHub, GitLab, or Bitbucket host; received: %s", parsedUrl.Host) } @@ -89,12 +77,13 @@ func parseGitUrlWithClient(fullUrl string, c http.Client) (GitUrl, error) { return g, err } -func parseGitHubUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { +func (g *GitUrl) parseGitHubUrl(url *url.URL) error { var splitUrl []string var err error g.Protocol = url.Scheme g.Host = url.Host + g.token = os.Getenv(GitHubToken) if g.Host == RawGitHubHost { g.IsFile = true @@ -131,20 +120,17 @@ func parseGitHubUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { } } - if !isGitUrlPublic(g, c) { - g.token = os.Getenv(GitHubToken) - } - - return g, err + return err } -func parseGitLabUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { +func (g *GitUrl) parseGitLabUrl(url *url.URL) error { var splitFile, splitOrg []string var err error g.Protocol = url.Scheme g.Host = url.Host g.IsFile = false + g.token = os.Getenv(GitLabToken) // GitLab urls contain a '-' separating the root of the repo // and the path to a file or directory @@ -175,20 +161,17 @@ func parseGitLabUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { } } - if !isGitUrlPublic(g, c) { - g.token = os.Getenv(GitLabToken) - } - - return g, err + return err } -func parseBitbucketUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { +func (g *GitUrl) parseBitbucketUrl(url *url.URL) error { var splitUrl []string var err error g.Protocol = url.Scheme g.Host = url.Host g.IsFile = false + g.token = os.Getenv(BitbucketToken) splitUrl = strings.SplitN(url.Path[1:], "/", 5) if len(splitUrl) < 2 { @@ -215,61 +198,51 @@ func parseBitbucketUrl(g GitUrl, url *url.URL, c http.Client) (GitUrl, error) { } } - if !isGitUrlPublic(g, c) { - g.token = os.Getenv(BitbucketToken) - } - - return g, err + return err } -func isGitUrlPublic(g GitUrl, c http.Client) bool { - host := g.Host - if host == RawGitHubHost { - host = GitHubHost - } - - repo := fmt.Sprintf("%s://%s/%s/%s", g.Protocol, host, g.Owner, g.Repo) - - if res, err := c.Get(repo); err != nil { - return false - } else if res.StatusCode == http.StatusOK { - return true - } - return false -} - -// CloneGitRepo clones a GitHub Repo to a destination directory +// CloneGitRepo clones a git repo to a destination directory func CloneGitRepo(g GitUrl, destDir string) error { - var cloneOptions *gitpkg.CloneOptions - host := g.Host if host == RawGitHubHost { host = GitHubHost } + isPublic := true repoUrl := fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, host, g.Owner, g.Repo) - branch := fmt.Sprintf("refs/heads/%s", g.Branch) - cloneOptions = &gitpkg.CloneOptions{ - URL: repoUrl, - ReferenceName: plumbing.ReferenceName(branch), - SingleBranch: true, - Depth: 1, + params := HTTPRequestParams{ + URL: repoUrl, } - if g.token != "" { - cloneOptions.Auth = &githttp.BasicAuth{ - // go-git auth allows username to be anything except - // an empty string for GitHub and GitLab, however requires - // for Bitbucket to be "x-token-auth" - Username: "x-token-auth", - Password: g.token, + // check if the git repo is public + _, err := HTTPGetRequest(params, 0) + if err != nil { + // private git repo requires authentication + isPublic = false + repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) + if g.Host == BitbucketHost { + repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", g.Protocol, g.token, host, g.Owner, g.Repo) } } - _, err := gitpkg.PlainClone(destDir, false, cloneOptions) + /* #nosec G204 -- user input is processed into an expected format for the git clone command */ + c := exec.Command("git", "clone", repoUrl, destDir) + c.Dir = destDir + + // set env to skip authentication prompt and directly error out + c.Env = os.Environ() + c.Env = append(c.Env, "GIT_TERMINAL_PROMPT=0", "GIT_ASKPASS=/bin/echo") + + _, err = c.CombinedOutput() if err != nil { - return err + if !isPublic { + if g.token == "" { + return fmt.Errorf("failed to clone repo without a token, ensure that a token is set if the repo is private. error: %v", err) + } + return fmt.Errorf("failed to clone repo with token, ensure that the url and token is correct. error: %v", err) + } + return fmt.Errorf("failed to clone repo, ensure that the url is correct. error: %v", err) } return nil diff --git a/pkg/util/git_test.go b/pkg/util/git_test.go index 2d62cc26..61e0e86f 100644 --- a/pkg/util/git_test.go +++ b/pkg/util/git_test.go @@ -1,5 +1,5 @@ // -// Copyright 2021-2022 Red Hat, Inc. +// Copyright 2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,8 +18,6 @@ package util import ( "github.com/kylelemons/godebug/pretty" "github.com/stretchr/testify/assert" - "io/ioutil" - "net/http" "os" "path/filepath" "reflect" @@ -32,22 +30,7 @@ var ( bitbucketToken = "fake-bitbucket-token" ) -type respondWithStatus struct { - status int -} - -func (rs respondWithStatus) RoundTrip(*http.Request) (*http.Response, error) { - return &http.Response{ - StatusCode: rs.status, - }, nil -} - -var ( - publicClient = http.Client{Transport: respondWithStatus{status: http.StatusOK}} - privateClient = http.Client{Transport: respondWithStatus{status: http.StatusNotFound}} -) - -func Test_parseGitUrlWithClient(t *testing.T) { +func Test_ParseGitUrl(t *testing.T) { defer func() { err := os.Unsetenv(githubToken) if err != nil { @@ -79,27 +62,23 @@ func Test_parseGitUrlWithClient(t *testing.T) { tests := []struct { name string url string - client http.Client wantUrl GitUrl wantErr string }{ { name: "should fail with empty url", url: "", - client: publicClient, wantErr: "URL is invalid", }, { name: "should fail with invalid git host", url: "https://google.ca/", - client: publicClient, wantErr: "url host should be a valid GitHub, GitLab, or Bitbucket host*", }, // GitHub { - name: "should parse public GitHub repo with root path", - url: "https://github.com/devfile/library", - client: publicClient, + name: "should parse public GitHub repo with root path", + url: "https://github.com/devfile/library", wantUrl: GitUrl{ Protocol: "https", Host: "github.com", @@ -107,20 +86,18 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "library", Branch: "", Path: "", - token: "", + token: "fake-github-token", IsFile: false, }, }, { name: "should fail with only GitHub host", url: "https://github.com/", - client: publicClient, wantErr: "url path should contain /*", }, { - name: "should parse public GitHub repo with file path", - url: "https://github.com/devfile/library/blob/main/devfile.yaml", - client: publicClient, + name: "should parse public GitHub repo with file path", + url: "https://github.com/devfile/library/blob/main/devfile.yaml", wantUrl: GitUrl{ Protocol: "https", Host: "github.com", @@ -128,14 +105,13 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "library", Branch: "main", Path: "devfile.yaml", - token: "", + token: "fake-github-token", IsFile: true, }, }, { - name: "should parse public GitHub repo with raw file path", - url: "https://raw.githubusercontent.com/devfile/library/main/devfile.yaml", - client: publicClient, + name: "should parse public GitHub repo with raw file path", + url: "https://raw.githubusercontent.com/devfile/library/main/devfile.yaml", wantUrl: GitUrl{ Protocol: "https", Host: "raw.githubusercontent.com", @@ -143,26 +119,23 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "library", Branch: "main", Path: "devfile.yaml", - token: "", + token: "fake-github-token", IsFile: true, }, }, { name: "should fail with missing GitHub repo", url: "https://github.com/devfile", - client: publicClient, wantErr: "url path should contain /*", }, { name: "should fail with invalid GitHub raw file path", url: "https://raw.githubusercontent.com/devfile/library/devfile.yaml", - client: publicClient, wantErr: "raw url path should contain ///*", }, { - name: "should parse private GitHub repo with token", - url: "https://github.com/fake-owner/fake-private-repo", - client: privateClient, + name: "should parse private GitHub repo with token", + url: "https://github.com/fake-owner/fake-private-repo", wantUrl: GitUrl{ Protocol: "https", Host: "github.com", @@ -175,9 +148,8 @@ func Test_parseGitUrlWithClient(t *testing.T) { }, }, { - name: "should parse private raw GitHub file path with token", - url: "https://raw.githubusercontent.com/fake-owner/fake-private-repo/main/README.md", - client: privateClient, + name: "should parse private raw GitHub file path with token", + url: "https://raw.githubusercontent.com/fake-owner/fake-private-repo/main/README.md", wantUrl: GitUrl{ Protocol: "https", Host: "raw.githubusercontent.com", @@ -191,9 +163,8 @@ func Test_parseGitUrlWithClient(t *testing.T) { }, // Gitlab { - name: "should parse public GitLab repo with root path", - url: "https://gitlab.com/gitlab-org/gitlab-foss", - client: publicClient, + name: "should parse public GitLab repo with root path", + url: "https://gitlab.com/gitlab-org/gitlab-foss", wantUrl: GitUrl{ Protocol: "https", Host: "gitlab.com", @@ -201,20 +172,18 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "gitlab-foss", Branch: "", Path: "", - token: "", + token: "fake-gitlab-token", IsFile: false, }, }, { name: "should fail with only GitLab host", url: "https://gitlab.com/", - client: publicClient, wantErr: "url path should contain /*", }, { - name: "should parse public GitLab repo with file path", - url: "https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/README.md", - client: publicClient, + name: "should parse public GitLab repo with file path", + url: "https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/README.md", wantUrl: GitUrl{ Protocol: "https", Host: "gitlab.com", @@ -222,26 +191,23 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "gitlab-foss", Branch: "master", Path: "README.md", - token: "", + token: "fake-gitlab-token", IsFile: true, }, }, { name: "should fail with missing GitLab repo", url: "https://gitlab.com/gitlab-org", - client: publicClient, wantErr: "url path should contain /*", }, { name: "should fail with missing GitLab keywords", url: "https://gitlab.com/gitlab-org/gitlab-foss/-/master/directory/README.md", - client: publicClient, wantErr: "url path should contain 'blob' or 'tree' or 'raw'*", }, { - name: "should parse private GitLab repo with token", - url: "https://gitlab.com/fake-owner/fake-private-repo", - client: privateClient, + name: "should parse private GitLab repo with token", + url: "https://gitlab.com/fake-owner/fake-private-repo", wantUrl: GitUrl{ Protocol: "https", Host: "gitlab.com", @@ -254,9 +220,8 @@ func Test_parseGitUrlWithClient(t *testing.T) { }, }, { - name: "should parse private raw GitLab file path with token", - url: "https://gitlab.com/fake-owner/fake-private-repo/-/raw/main/README.md", - client: privateClient, + name: "should parse private raw GitLab file path with token", + url: "https://gitlab.com/fake-owner/fake-private-repo/-/raw/main/README.md", wantUrl: GitUrl{ Protocol: "https", Host: "gitlab.com", @@ -270,9 +235,8 @@ func Test_parseGitUrlWithClient(t *testing.T) { }, // Bitbucket { - name: "should parse public Bitbucket repo with root path", - url: "https://bitbucket.org/fake-owner/fake-public-repo", - client: publicClient, + name: "should parse public Bitbucket repo with root path", + url: "https://bitbucket.org/fake-owner/fake-public-repo", wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", @@ -280,20 +244,18 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "fake-public-repo", Branch: "", Path: "", - token: "", + token: "fake-bitbucket-token", IsFile: false, }, }, { name: "should fail with only Bitbucket host", url: "https://bitbucket.org/", - client: publicClient, wantErr: "url path should contain /*", }, { - name: "should parse public Bitbucket repo with file path", - url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/README.md", - client: publicClient, + name: "should parse public Bitbucket repo with file path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/README.md", wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", @@ -301,14 +263,13 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "fake-public-repo", Branch: "main", Path: "README.md", - token: "", + token: "fake-bitbucket-token", IsFile: true, }, }, { - name: "should parse public Bitbucket file path with nested path", - url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/directory/test.txt", - client: publicClient, + name: "should parse public Bitbucket file path with nested path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/directory/test.txt", wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", @@ -316,14 +277,13 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "fake-public-repo", Branch: "main", Path: "directory/test.txt", - token: "", + token: "fake-bitbucket-token", IsFile: true, }, }, { - name: "should parse public Bitbucket repo with raw file path", - url: "https://bitbucket.org/fake-owner/fake-public-repo/raw/main/README.md", - client: publicClient, + name: "should parse public Bitbucket repo with raw file path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/raw/main/README.md", wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", @@ -331,32 +291,28 @@ func Test_parseGitUrlWithClient(t *testing.T) { Repo: "fake-public-repo", Branch: "main", Path: "README.md", - token: "", + token: "fake-bitbucket-token", IsFile: true, }, }, { name: "should fail with missing Bitbucket repo", url: "https://bitbucket.org/fake-owner", - client: publicClient, wantErr: "url path should contain /*", }, { name: "should fail with invalid Bitbucket directory or file path", url: "https://bitbucket.org/fake-owner/fake-public-repo/main/README.md", - client: publicClient, wantErr: "url path should contain path to directory or file*", }, { name: "should fail with missing Bitbucket keywords", url: "https://bitbucket.org/fake-owner/fake-public-repo/main/test/README.md", - client: publicClient, wantErr: "url path should contain 'raw' or 'src'*", }, { - name: "should parse private Bitbucket repo with token", - url: "https://bitbucket.org/fake-owner/fake-private-repo", - client: privateClient, + name: "should parse private Bitbucket repo with token", + url: "https://bitbucket.org/fake-owner/fake-private-repo", wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", @@ -369,9 +325,8 @@ func Test_parseGitUrlWithClient(t *testing.T) { }, }, { - name: "should parse private raw Bitbucket file path with token", - url: "https://bitbucket.org/fake-owner/fake-private-repo/raw/main/README.md", - client: privateClient, + name: "should parse private raw Bitbucket file path with token", + url: "https://bitbucket.org/fake-owner/fake-private-repo/raw/main/README.md", wantUrl: GitUrl{ Protocol: "https", Host: "bitbucket.org", @@ -387,7 +342,7 @@ func Test_parseGitUrlWithClient(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseGitUrlWithClient(tt.url, tt.client) + got, err := ParseGitUrl(tt.url) if (err != nil) != (tt.wantErr != "") { t.Errorf("Unxpected error: %t, want: %v", err, tt.wantUrl) } else if err == nil && !reflect.DeepEqual(got, tt.wantUrl) { @@ -400,11 +355,9 @@ func Test_parseGitUrlWithClient(t *testing.T) { } func TestCloneGitRepo(t *testing.T) { - tempDir, err := ioutil.TempDir("", "") - if err != nil { - t.Errorf("Failed to create temp dir: %s, error: %v", tempDir, err) - } - defer os.RemoveAll(tempDir) + tempDirGitHub := t.TempDir() + tempDirGitLab := t.TempDir() + tempDirBitbucket := t.TempDir() invalidGitUrl := GitUrl{ Protocol: "", @@ -422,31 +375,76 @@ func TestCloneGitRepo(t *testing.T) { Branch: "main", } + validGitLabUrl := GitUrl{ + Protocol: "https", + Host: "gitlab.com", + Owner: "mike-hoang", + Repo: "public-testing-repo", + Branch: "main", + } + + validBitbucketUrl := GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "mike-hoang", + Repo: "public-testing-repo", + Branch: "master", + } + + privateGitHubRepo := GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "fake-owner", + Repo: "fake-private-repo", + Branch: "master", + token: "fake-github-token", + } + + privateRepoMissingTokenErr := "failed to clone repo without a token*" + privateRepoBadTokenErr := "failed to clone repo with token*" + tests := []struct { name string gitUrl GitUrl destDir string - wantErr bool + wantErr string }{ { name: "should fail with invalid git url", gitUrl: invalidGitUrl, destDir: filepath.Join(os.TempDir(), "nonexistent"), - wantErr: true, + wantErr: privateRepoMissingTokenErr, }, { - name: "should be able to clone valid github url", + name: "should fail to clone valid private git url with a bad token", + gitUrl: privateGitHubRepo, + destDir: filepath.Join(os.TempDir(), "nonexistent"), + wantErr: privateRepoBadTokenErr, + }, + { + name: "should be able to clone valid public github url", gitUrl: validGitHubUrl, - destDir: tempDir, - wantErr: false, + destDir: tempDirGitHub, + }, + { + name: "should be able to clone valid public gitlab url", + gitUrl: validGitLabUrl, + destDir: tempDirGitLab, + }, + { + name: "should be able to clone valid public bitbucket url", + gitUrl: validBitbucketUrl, + destDir: tempDirBitbucket, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := CloneGitRepo(tt.gitUrl, tt.destDir) - if (err != nil) != tt.wantErr { - t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err) + if (err != nil) != (tt.wantErr != "") { + t.Errorf("Unxpected error: %t, want: %v", err, tt.wantErr) + } else if err != nil { + assert.Regexp(t, tt.wantErr, err.Error(), "Error message should match") } }) }