Skip to content

Commit

Permalink
add support for selfhosted gitlab/github
Browse files Browse the repository at this point in the history
  • Loading branch information
@kloenk
kloenk committed Jun 4, 2020
1 parent 81cafda commit be48d23
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 15 deletions.
51 changes: 36 additions & 15 deletions src/libfetchers/github.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,15 @@

namespace nix::fetchers {

// get the default url attribute
std::string getDefaultUrl(const Attrs & attrs, const std::string & url) {
auto s = maybeGetStrAttr(attrs, "url");
if (!s)
return url;
return *s;
}


struct GitArchiveInputScheme : InputScheme
{
virtual std::string type() = 0;
Expand All @@ -20,6 +29,7 @@ struct GitArchiveInputScheme : InputScheme

std::optional<Hash> rev;
std::optional<std::string> ref;
std::optional<std::string> host_url;

if (path.size() == 2) {
} else if (path.size() == 3) {
Expand All @@ -45,6 +55,11 @@ struct GitArchiveInputScheme : InputScheme
throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
ref = value;
}
else if (name == "url") {
if (!std::regex_match(value, urlRegex))
throw BadURL("URL '%s' contains an invalid instance url", url.url);
host_url = value;
}
// FIXME: barf on unsupported attributes
}

Expand All @@ -57,6 +72,7 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.insert_or_assign("repo", path[1]);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) input.attrs.insert_or_assign("ref", *ref);
if (host_url) input.attrs.insert_or_assign("url", *host_url);

return input;
}
Expand Down Expand Up @@ -171,8 +187,9 @@ struct GitHubInputScheme : GitArchiveInputScheme

Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
auto host_url = getDefaultUrl(input.attrs, "github.com");
auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
Expand All @@ -186,9 +203,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
{
// FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits.

auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
auto host_url = getDefaultUrl(input.attrs, "github.com");
auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));

std::string accessToken = settings.githubAccessToken.get();
Expand All @@ -200,8 +217,9 @@ struct GitHubInputScheme : GitArchiveInputScheme

void clone(const Input & input, const Path & destDir) override
{
Input::fromURL(fmt("git+ssh://[email protected]/%s/%s.git",
getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
auto host_url = getDefaultUrl(input.attrs, "github.com");
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("master"), input.getRev())
.clone(destDir);
}
Expand All @@ -213,8 +231,9 @@ struct GitLabInputScheme : GitArchiveInputScheme

Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/branches/%s",
getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
auto host_url = getDefaultUrl(input.attrs, "gitlab.com");
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/branches/%s",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
Expand All @@ -226,10 +245,10 @@ struct GitLabInputScheme : GitArchiveInputScheme

std::string getDownloadUrl(const Input & input) const override
{
// FIXME: This endpoint has a rate limit threshold of 5 requests per minute.

auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
// FIXME: This endpoint has a rate limit threshold of 5 requests per minute
auto host_url = getDefaultUrl(input.attrs, "gitlab.com");
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));

/* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`)
Expand All @@ -242,8 +261,10 @@ struct GitLabInputScheme : GitArchiveInputScheme

void clone(const Input & input, const Path & destDir) override
{
Input::fromURL(fmt("git+ssh://[email protected]/%s/%s.git",
getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
auto host_url = getDefaultUrl(input.attrs, "gitlab.com");
// FIXME: get username somewhere
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("master"), input.getRev())
.clone(destDir);
}
Expand Down
1 change: 1 addition & 0 deletions src/libutil/url.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ std::regex refRegex(refRegexS, std::regex::ECMAScript);
std::regex badGitRefRegex(badGitRefRegexS, std::regex::ECMAScript);
std::regex revRegex(revRegexS, std::regex::ECMAScript);
std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript);
std::regex urlRegex(urlRegexS, std::regex::ECMAScript);

ParsedURL parseURL(const std::string & url)
{
Expand Down
4 changes: 4 additions & 0 deletions src/libutil/url.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRege
const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
extern std::regex refRegex;

// A github or gitlab url
const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
extern std::regex urlRegex;

// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
// This is because of the definition of a ref in refs.c in https://github.com/git/git
// See tests/fetchGitRefs.sh for the full definition
Expand Down

0 comments on commit be48d23

Please sign in to comment.