diff --git a/haxelib.json b/haxelib.json index 9da66e387..527508941 100644 --- a/haxelib.json +++ b/haxelib.json @@ -7,5 +7,8 @@ "classPath": "src", "version": "4.0.2", "releasenote": " * Fixed too strict requirements to haxelib.json data for private libs (#484)", - "contributors": ["HaxeFoundation", "back2dos", "ncannasse", "jason", "Simn", "nadako", "andyli"] + "contributors": ["HaxeFoundation", "back2dos", "ncannasse", "jason", "Simn", "nadako", "andyli"], + "dependencies":{ + "hx3compat":"git:https://github.com/haxefoundation/hx3compat.git#f1f18201e5c0479cb5adf5f6028788b37f37b730" + } } diff --git a/src/haxelib/Data.hx b/src/haxelib/Data.hx index 2a186e0fa..0ec26035a 100644 --- a/src/haxelib/Data.hx +++ b/src/haxelib/Data.hx @@ -131,7 +131,7 @@ abstract Dependencies(Dynamic) from Dynamic) from Dynamic= 4.0) + public inline function keyValueIterator():KeyValueIterator { + final fields = Reflect.fields(this); + var index = 0; + return { + next: function() { + final name = fields[index++]; + return {key: ProjectName.ofString(name), value: Reflect.field(this, name)}; + }, + hasNext: function() return index < fields.length + } + } + #end + + /** Returns an array of the names of the dependencies. **/ + public inline function getNames():Array + return [for(name in Reflect.fields(this)) ProjectName.ofString(name) ]; + } /** The type of a dependency version. **/ @@ -212,75 +232,6 @@ typedef Infos = { var Apache = 'Apache'; } -/** A valid project name string. **/ -abstract ProjectName(String) to String { - static var RESERVED_NAMES = ["haxe", "all"]; - static var RESERVED_EXTENSIONS = ['.zip', '.hxml']; - inline function new(s:String) - this = s; - - @:to function toValidatable():Validatable - return { - validate: - function ():Option { - for (r in rules) - if (!r.check(this)) - return Some(r.msg.replace('%VALUE', '`' + Json.stringify(this) + '`')); - return None; - } - } - - static var rules = {//using an array because order might matter - var a = new Array<{ msg: String, check:String->Bool }>(); - - function add(m, r) - a.push( { msg: m, check: r } ); - - add("%VALUE is not a String", - #if (haxe_ver < 4.1) - Std.is.bind(_, String) - #else - Std.isOfType.bind(_, String) - #end - ); - add("%VALUE is too short", function (s) return s.length >= 3); - add("%VALUE contains invalid characters", Data.alphanum.match); - add("%VALUE is a reserved name", function(s) return RESERVED_NAMES.indexOf(s.toLowerCase()) == -1); - add("%VALUE ends with a reserved suffix", function(s) { - s = s.toLowerCase(); - for (ext in RESERVED_EXTENSIONS) - if (s.endsWith(ext)) return false; - return true; - }); - - a; - } - - /** - Validates that the project name is valid. - - If it is invalid, returns `Some(e)` where e is an error - detailing why the project name is invalid. - - If it is valid, returns `None`. - **/ - public function validate() - return toValidatable().validate(); - - /** - Returns `s` as a `ProjectName` if it is valid, - otherwise throws an error explaining why it is invalid. - **/ - static public function ofString(s:String) - return switch new ProjectName(s) { - case _.toValidatable().validate() => Some(e): throw e; - case v: v; - } - - /** Default project name **/ - static public var DEFAULT(default, null) = new ProjectName('unknown'); -} - /** Class providing functions for working with project information. **/ class Data { @@ -399,15 +350,22 @@ class Data { } } - /** Extracts project information from `jsondata`, validating it according to `check`. **/ - public static function readData( jsondata: String, check : CheckLevel ) : Infos { + /** + Extracts project information from `jsondata`, validating it according to `check`. + + `defaultName` is the project name to use if it is empty when the check value allows it. + **/ + public static function readData( jsondata: String, check : CheckLevel, ?defaultName:ProjectName ) : Infos { + if (defaultName == null) + defaultName = ProjectName.DEFAULT; + var doc:Infos = try Json.parse(jsondata) catch ( e : Dynamic ) if (check >= CheckLevel.CheckSyntax) throw 'JSON parse error: $e'; else { - name : ProjectName.DEFAULT, + name : defaultName, url : '', version : SemVer.DEFAULT, releasenote: 'No haxelib.json found', @@ -436,7 +394,7 @@ class Data { doc.classPath = ''; if (doc.name.validate() != None) - doc.name = ProjectName.DEFAULT; + doc.name = defaultName; if (doc.description == null) doc.description = ''; diff --git a/src/haxelib/ProjectName.hx b/src/haxelib/ProjectName.hx new file mode 100644 index 000000000..8d113e25d --- /dev/null +++ b/src/haxelib/ProjectName.hx @@ -0,0 +1,82 @@ +package haxelib; + +import haxe.Json; +import haxe.ds.Option; + +import haxelib.Validator; + +using StringTools; + +/** A valid project name string. **/ +abstract ProjectName(String) to String { + static var RESERVED_NAMES = ["haxe", "all"]; + static var RESERVED_EXTENSIONS = ['.zip', '.hxml']; + + inline function new(s:String) + this = s; + + @:to function toValidatable():Validatable + return { + validate: function():Option { + for (r in rules) + if (!r.check(this)) + return Some(r.msg.replace('%VALUE', '`' + Json.stringify(this) + '`')); + return None; + } + } + + static var rules = { // using an array because order might matter + var a = new Array<{msg:String, check:String->Bool}>(); + function add(m, r) + a.push({msg: m, check: r}); + add("%VALUE is not a String", #if (haxe_ver < 4.1) Std.is .bind(_, String) #else Std.isOfType.bind(_, String) #end + ); + add("%VALUE is too short", function(s) return s.length >= 3); + add("%VALUE contains invalid characters", Data.alphanum.match); + add("%VALUE is a reserved name", function(s) return RESERVED_NAMES.indexOf(s.toLowerCase()) == -1); + add("%VALUE ends with a reserved suffix", function(s) { + s = s.toLowerCase(); + for (ext in RESERVED_EXTENSIONS) + if (s.endsWith(ext)) + return false; + return true; + }); + a; + } + + /** + Validates that the project name is valid. + + If it is invalid, returns `Some(e)` where e is an error + detailing why the project name is invalid. + + If it is valid, returns `None`. + **/ + public function validate() + return toValidatable().validate(); + + public function toLowerCase():ProjectName + return new ProjectName(this.toLowerCase()); + + /** + Returns `s` as a `ProjectName` if it is valid, + otherwise throws an error explaining why it is invalid. + **/ + static public function ofString(s:String) + return switch new ProjectName(s) { + case _.toValidatable().validate() => Some(e): throw e; + case v: v; + } + + /** + If `alias` is just a different capitalization of `correct`, returns `correct`. + + If `alias` is completely different, returns `alias` instead. + **/ + static public function getCorrectOrAlias(correct:ProjectName, alias:ProjectName) { + return if (correct.toLowerCase() == alias.toLowerCase()) correct else alias; + } + + /** Default project name **/ + static public var DEFAULT(default, null) = new ProjectName('unknown'); +} diff --git a/src/haxelib/Util.hx b/src/haxelib/Util.hx new file mode 100644 index 000000000..d9603851b --- /dev/null +++ b/src/haxelib/Util.hx @@ -0,0 +1,60 @@ +package haxelib; + +#if macro +import haxe.macro.Expr; + +using haxe.macro.Tools; +#end + +using StringTools; + +class Util { + macro static public function rethrow(e) { + return if (haxe.macro.Context.defined("neko")) + macro neko.Lib.rethrow(e); + else + macro throw e; + } + + #if macro + static function readVersionFromHaxelibJson() { + return haxe.Json.parse(sys.io.File.getContent("haxelib.json")).version; + } + #end + + macro static public function getHaxelibVersion() { + return macro $v{readVersionFromHaxelibJson()}; + } + + macro static public function getHaxelibVersionLong() { + var version:String = readVersionFromHaxelibJson(); + var p; + try { + //check if the .git folder exist + //prevent getting the git info of a parent directory + if (!sys.FileSystem.isDirectory(".git")) + throw "Not a git repo."; + + //get commit sha + p = new sys.io.Process("git", ["rev-parse", "HEAD"]); + final sha = p.stdout.readAll().toString().trim(); + p.close(); + + //check to see if there is changes, staged or not + p = new sys.io.Process("git", ["status", "--porcelain"]); + final changes = p.stdout.readAll().toString().trim(); + p.close(); + + version += switch(changes) { + case "": + ' ($sha)'; + case _: + ' ($sha - dirty)'; + } + return macro $v{version}; + } catch(e:Dynamic) { + if (p != null) p.close(); + return macro $v{version}; + } + } +} diff --git a/src/haxelib/api/Connection.hx b/src/haxelib/api/Connection.hx new file mode 100644 index 000000000..9c6e3b81f --- /dev/null +++ b/src/haxelib/api/Connection.hx @@ -0,0 +1,544 @@ +package haxelib.api; + +import haxe.Http; +import haxe.Timer; +import haxe.zip.*; +import haxe.io.BytesOutput; +import haxe.io.Output; +import haxe.io.Input; +import haxe.remoting.HttpConnection; +import sys.FileSystem; +import sys.io.File; + +import haxelib.Data; + +using StringTools; + +#if js +import haxe.io.Bytes; +using haxelib.api.Connection.PromiseSynchronizer; + +@:jsRequire("promise-synchronizer") +private extern class PromiseSynchronizer { + @:selfCall + static public function sync(p:js.lib.Promise):T; +} +#end + +private class SiteProxy extends haxe.remoting.Proxy {} + +@:structInit +private class ServerInfo { + public final protocol:String; + public final host:String; + public final port:Int; + public final dir:String; + public final url:String; + public final apiVersion:String; + public final useSsl:Bool; +} + +@:structInit +private class ConnectionData { + public final site:SiteProxy; + public final server:ServerInfo; + public final siteUrl:String; + + public static function setup(remote:String = null, useSsl = true):ConnectionData { + final server = switch remote { + case null: getDefault(useSsl); + case remote: getFromRemote(remote, useSsl); + } + final siteUrl = '${server.protocol}://${server.host}:${server.port}/${server.dir}'; + + final remotingUrl = '${siteUrl}api/${server.apiVersion}/${server.url}'; + final site = new SiteProxy(HttpConnection.urlConnect(remotingUrl).resolve("api")); + + return { + site: site, + server: server, + siteUrl: siteUrl + } + } + + static function getDefault(useSsl:Bool):ServerInfo { + return { + protocol: useSsl ? "https" : "http", + host: "lib.haxe.org", + port: useSsl ? 443 : 80, + dir: "", + url: "index.n", + apiVersion: "3.0", + useSsl: useSsl + }; + } + + static function getFromRemote(remote:String, useSsl:Bool):ServerInfo { + final r = ~/^(?:(https?):\/\/)?([^:\/]+)(?::([0-9]+))?\/?(.*)$/; + if (!r.match(remote)) + throw 'Invalid repository format \'$remote\''; + + final protocol = r.matched(1) ?? (if (useSsl) "https" else "http"); + + final port = switch (r.matched(3)) { + case null if (protocol == "https"): 443; + case null if (protocol == "http"): 80; + case null: throw 'unknown default port for $protocol'; + case Std.parseInt(_) => port if(port != null): port; + case invalidPortStr: throw '$invalidPortStr is not a valid port'; + } + + return { + protocol: protocol, + host: r.matched(2), + port: port, + dir: haxe.io.Path.addTrailingSlash(r.matched(4)), + url: "index.n", + apiVersion: "3.0", + useSsl: useSsl + }; + } +} + +/** Signature of function used to log the progress of a download. **/ +@:noDoc +typedef DownloadProgress = (finished:Bool, cur:Int, max:Null, downloaded:Int, time:Float) -> Void; + +private class ProgressOut extends Output { + final o:Output; + final startSize:Int; + final start:Float; + + var cur:Int; + var max:Null; + + public function new(o, currentSize, progress) { + start = Timer.stamp(); + this.o = o; + startSize = currentSize; + cur = currentSize; + this.progress = progress; + } + + dynamic function progress(finished:Bool, cur:Int, max:Null, downloaded:Int, time:Float):Void {} + + function report(n) { + cur += n; + + progress(false, cur, max, cur - startSize, Timer.stamp() - start); + } + + public override function writeByte(c) { + o.writeByte(c); + report(1); + } + + public override function writeBytes(s, p, l) { + final r = o.writeBytes(s, p, l); + report(r); + return r; + } + + public override function close() { + super.close(); + o.close(); + + progress(true, cur, max, cur - startSize, Timer.stamp() - start); + } + + public override function prepare(m) { + max = m + startSize; + } +} + +private class ProgressIn extends Input { + final i:Input; + final tot:Int; + + var pos:Int; + + public function new(i, tot, progress) { + this.i = i; + this.pos = 0; + this.tot = tot; + this.progress = progress; + } + + dynamic function progress(pos:Int, total:Int):Void {} + + public override function readByte() { + final c = i.readByte(); + report(1); + return c; + } + + public override function readBytes(buf, pos, len) { + final k = i.readBytes(buf, pos, len); + report(k); + return k; + } + + function report(nbytes:Int) { + pos += nbytes; + progress(pos,tot); + } +} + +/** Class that provides functions for interactions with the Haxelib server. **/ +class Connection { + /** The number of times a server interaction will be attempted. Defaults to 3. **/ + public static var retries = 3; + + /** If set to `false`, the connection timeout time is unlimited. **/ + public static var hasTimeout(default, set) = true; + + static function set_hasTimeout(value:Bool) { + if (value) + haxe.remoting.HttpConnection.TIMEOUT = 10; + else + haxe.remoting.HttpConnection.TIMEOUT = 0; + return hasTimeout = value; + } + /** Whether to use SSL when connecting. Set to `true` by default. **/ + public static var useSsl(default, set) = true; + static function set_useSsl(value:Bool):Bool { + if (useSsl != value) + data = null; + return useSsl = value; + } + + /** The server url to be used as the Haxelib database. **/ + public static var remote(default, set):Null = null; + static function set_remote(value:String):String { + if (remote != value) + data = null; + return remote = value; + } + + /** Function to which connection information will be logged. **/ + public static dynamic function log(msg:String) {} + + /** Returns the domain of the Haxelib server. **/ + public static function getHost():String { + return data.server.host; + } + + static var data(get, null):ConnectionData; + static function get_data():ConnectionData { + if (data == null) + return data = ConnectionData.setup(remote, useSsl); + return data; + } + + /** + Downloads the file from `fileUrl` into `outpath`. + + `downloadProgress` is the function used to log download information. + **/ + #if js + public static function download(fileUrl:String, outPath:String, downloadProgress = null):Void { + node_fetch.Fetch.call(fileUrl, { + headers: { + "User-Agent": 'haxelib ${Util.getHaxelibVersionLong()}', + } + }) + .then(r -> r.ok ? r.arrayBuffer() : throw 'Request to $fileUrl responded with ${r.statusText}') + .then(buf -> File.saveBytes(outPath, Bytes.ofData(buf))) + .sync(); + } + #else + public static function download(filename:String, outPath:String, downloadProgress:DownloadProgress = null) { + final maxRetry = 3; + final fileUrl = haxe.io.Path.join([data.siteUrl, Data.REPOSITORY, filename]); + var lastError = new haxe.Exception(""); + + for (i in 0...maxRetry) { + try { + downloadFromUrl(fileUrl, outPath, downloadProgress); + return; + } catch (e:Dynamic) { + log('Failed to download ${fileUrl}. (${i + 1}/${maxRetry})\n${e}'); + lastError = e; + Sys.sleep(1); + } + } + FileSystem.deleteFile(outPath); + throw lastError; + } + + // maxRedirect set to 20, which is most browsers' default value according to https://stackoverflow.com/a/36041063/267998 + static function downloadFromUrl(fileUrl:String, outPath:String, downloadProgress:Null, maxRedirect = 20):Void { + final out = try File.append(outPath, true) catch (e:Dynamic) throw 'Failed to write to $outPath: $e'; + out.seek(0, SeekEnd); + + final h = createHttpRequest(fileUrl); + + final currentSize = out.tell(); + if (currentSize > 0) + h.addHeader("range", 'bytes=$currentSize-'); + + final progress = if (downloadProgress == null) out + else new ProgressOut(out, currentSize, downloadProgress); + + var httpStatus = -1; + var redirectedLocation = null; + h.onStatus = function(status) { + httpStatus = status; + switch (httpStatus) { + case 301, 302, 307, 308: + switch (h.responseHeaders.get("Location")) { + case null: + throw 'Request to $fileUrl responded with $httpStatus, ${h.responseHeaders}'; + case location: + redirectedLocation = location; + } + default: + // TODO? + } + }; + h.onError = function(e) { + progress.close(); + + switch (httpStatus) { + case 416: + // 416 Requested Range Not Satisfiable, which means that we probably have a fully downloaded file already + // if we reached onError, because of 416 status code, it's probably okay and we should try unzipping the file + default: + FileSystem.deleteFile(outPath); + throw e; + } + }; + h.customRequest(false, progress); + + if (redirectedLocation == null) + return; + + FileSystem.deleteFile(outPath); + + if (maxRedirect == 0) + throw "Too many redirects."; + + downloadFromUrl(redirectedLocation, outPath, downloadProgress, maxRedirect - 1); + } + #end + + static function retry(func:Void->R) { + var hasRetried = false; + var numTries = retries; + + while (numTries-- > 0) { + try { + final result = func(); + + if (hasRetried) + log("Retry successful"); + + return result; + } catch (e:Dynamic) { + if (e == "std@host_resolve") + Util.rethrow(e); + if (e != "Blocked") + throw 'Failed with error: $e'; + log("Failed. Triggering retry due to HTTP timeout"); + hasRetried = true; + } + } + throw 'Failed due to HTTP timeout after multiple retries'; + } + + /** Returns the array of available versions for `library`. **/ + static function getVersions(library:ProjectName):Array { + final versionsData = retry(data.site.infos.bind(library)).versions; + return [for (data in versionsData) data.name]; + } + + /** + Returns a map of the library names in `libraries` with their validated names + (to ensure correct capitalisation) and available versions. + **/ + public static function getVersionsForLibraries(libraries:Array):Map}> { + // TODO: can we collapse this into a single API call? It's getting too slow otherwise. + final map = new Map}>(); + + for (lib in libraries) { + final info = retry(data.site.infos.bind(lib)); + final versionsData = info.versions; + // TODO with stricter capitalisation we won't have to use info.name maybe + map[lib] = {confirmedName: ProjectName.ofString(info.name), versions:[for(data in versionsData) data.name]}; + } + return map; + } + + #if !js + /** + Submits the library at `path` and uploads it. + + `login` is called with the project's contributors and expects one of them + to be returned along with the password. + + If the library version being submitted already exists, `overwrite` is called + with the library version, and the version is overwritten only if it returns `true`, + otherwise the operation is aborted. If `overwrite` is not passed in, the + operation is aborted by default. + + `logUploadStatus` can be passed in optionally to show progress during upload. + **/ + public static function submitLibrary(path:String, login:(Array)->{name:String, password:String}, + ?overwrite:(version:SemVer)->Bool, + ?logUploadStatus:(current:Int, total:Int) -> Void + ) { + var data:haxe.io.Bytes, zip:List; + if (FileSystem.isDirectory(path)) { + zip = FsUtils.zipDirectory(path); + final out = new BytesOutput(); + new Writer(out).write(zip); + data = out.getBytes(); + } else { + data = File.getBytes(path); + zip = Reader.readZip(new haxe.io.BytesInput(data)); + } + + final infos = Data.readInfos(zip, true); + Data.checkClassPath(zip, infos); + + // ask user which contributor they are + final user = login(infos.contributors); + // ensure they are already a contributor for the latest release + checkDeveloper(infos.name, user.name); + + checkDependencies(infos.dependencies); + + // check if this version already exists + if (doesVersionExist(infos.name, infos.version) && !(overwrite == null || overwrite(infos.version))) + throw "Aborted"; + + uploadAndSubmit(user, data, logUploadStatus); + } + + static function checkDependencies(dependencies:Dependencies) { + for (d in dependencies) { + final versions:Array = getVersions(ProjectName.ofString(d.name)); + if (d.version == "") + continue; + if (!versions.contains(d.version)) + throw "Library " + d.name + " does not have version " + d.version; + } + } + + static function doesVersionExist(library:ProjectName, version:SemVer):Bool { + final versions = try getVersions(library) catch (_:Dynamic) return false; + return versions.contains(version); + } + + static function uploadAndSubmit(user, data, uploadProgress:Null<(pos:Int, total:Int) -> Void>) { + // query a submit id that will identify the file + final id = getSubmitId(); + + upload(data, id, uploadProgress); + + log("Processing file..."); + + // processing might take some time, make sure we wait + final oldTimeout = HttpConnection.TIMEOUT; + if (hasTimeout) // don't ignore `hasTimeout` being false + HttpConnection.TIMEOUT = 1000; + + // ask the server to register the sent file + final msg = processSubmit(id, user.name, user.password); + log(msg); + + HttpConnection.TIMEOUT = oldTimeout; + } + + static function upload(data:haxe.io.Bytes, id:String, logUploadStatus:Null<(pos:Int, total:Int) -> Void>) { + // directly send the file data over Http + final h = createRequest(); + h.onError = function(e) throw e; + h.onData = log; + + final inp = { + final dataBytes = new haxe.io.BytesInput(data); + if (logUploadStatus == null) + dataBytes; + new ProgressIn(dataBytes, data.length, logUploadStatus); + } + + h.fileTransfer("file", id, inp, data.length); + log("Sending data..."); + h.request(true); + } + + static inline function createRequest():Http { + return createHttpRequest('${data.server.protocol}://${data.server.host}:${data.server.port}/${data.server.url}'); + } + + static inline function createHttpRequest(url:String):Http { + final req = new Http(url); + req.addHeader("User-Agent", "haxelib " + Util.getHaxelibVersionLong()); + if (!hasTimeout) + req.cnxTimeout = 0; + return req; + } + + /** Sets the proxy that will be used for http requests **/ + public static function setProxy(proxy:{host:String, port:Null, auth:Null<{user:String, pass:String}>}):Void { + Http.PROXY = proxy; + } + + /** + Makes a connection attempt across the internet, and returns `true` + if connection is successful, or `false` if it fails. + **/ + public static function testConnection():Bool { + try { + Http.requestUrl(data.server.protocol + "://lib.haxe.org"); + return true; + } catch (e:Dynamic) { + return false; + } + } + #end + + // Could maybe be done with a macro ?? + + /** Returns the latest version of `library`. **/ + public static function getLatestVersion(library:ProjectName):SemVer + return retry(data.site.getLatestVersion.bind(library)); + + /** Returns the project information of `library` **/ + public static function getInfo(library:ProjectName):ProjectInfos + return retry(data.site.infos.bind(library)); + + /** Searches libraries with `word` as the search term. **/ + public static function search(word:String) + return retry(data.site.search.bind(word)); + + /** Informs the server of a successful installation of `version` of `library` **/ + public static function postInstall(library:ProjectName, version:SemVer) + return retry(data.site.postInstall.bind(library, version)); + + /** Returns user information for `userName` **/ + public static function getUserData(userName:String) + return retry(data.site.user.bind(userName)); + + /** Registers user with `name`, `encodedPassword`, `email`, and `fullname`. **/ + public static function register(name:String, encodedPassword:String, email:String, fullname:String) + return retry(data.site.register.bind(name, encodedPassword, email, fullname)); + + /** Returns `true` if no user with `userName` exists yet. **/ + public static function isNewUser(userName:String) + return retry(data.site.isNewUser.bind(userName)); + + /** Checks that `password` is the correct password for `userName`. **/ + public static function checkPassword(userName:String, password:String) + return retry(data.site.checkPassword.bind(userName, password)); + + static function checkDeveloper(library:ProjectName, userName:String) + return retry(data.site.checkDeveloper.bind(library, userName)); + + static function getSubmitId() + return retry(data.site.getSubmitId.bind()); + + static function processSubmit(id:String, userName:String, password:String) + return retry(data.site.processSubmit.bind(id, userName, password)); +} diff --git a/src/haxelib/client/ConvertXml.hx b/src/haxelib/api/ConvertXml.hx similarity index 99% rename from src/haxelib/client/ConvertXml.hx rename to src/haxelib/api/ConvertXml.hx index 0e1206d03..f44d3c454 100644 --- a/src/haxelib/client/ConvertXml.hx +++ b/src/haxelib/api/ConvertXml.hx @@ -19,7 +19,7 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ -package haxelib.client; +package haxelib.api; import haxe.Json; diff --git a/src/haxelib/client/FsUtils.hx b/src/haxelib/api/FsUtils.hx similarity index 69% rename from src/haxelib/client/FsUtils.hx rename to src/haxelib/api/FsUtils.hx index 6cbeed825..778c4296e 100644 --- a/src/haxelib/client/FsUtils.hx +++ b/src/haxelib/api/FsUtils.hx @@ -19,17 +19,21 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ -package haxelib.client; +package haxelib.api; -import haxe.io.Path; import sys.FileSystem; +import sys.io.File; +import haxe.io.Path; +import haxe.zip.Reader; +import haxe.zip.Entry; +import haxe.zip.Tools; + using StringTools; /** Class containing useful FileSystem utility functions. **/ @:noDoc class FsUtils { - static var IS_WINDOWS = (Sys.systemName() == "Windows"); - + public static final IS_WINDOWS = (Sys.systemName() == "Windows"); /** Recursively follow symlink @@ -140,4 +144,76 @@ class FsUtils { try FileSystem.fullPath(path) catch (error:String) if (error == "std@file_full_path") errors++; return errors == 2; } + + public static function getHomePath():String { + var home:String = null; + if (IS_WINDOWS) { + home = Sys.getEnv("USERPROFILE"); + if (home == null) { + final drive = Sys.getEnv("HOMEDRIVE"); + final path = Sys.getEnv("HOMEPATH"); + if (drive != null && path != null) + home = drive + path; + } + if (home == null) + throw "Could not determine home path. Please ensure that USERPROFILE or HOMEDRIVE+HOMEPATH environment variables are set."; + } else { + home = Sys.getEnv("HOME"); + if (home == null) + throw "Could not determine home path. Please ensure that HOME environment variable is set."; + } + return home; + } + + /** Unzips the file at `filePath`, but if an error is thrown, it will safely close the file before rethrowing. **/ + public static function unzip(filePath:String):List { + final file = sys.io.File.read(filePath, true); + try { + final zip = Reader.readZip(file); + file.close(); + return zip; + } catch (e:Dynamic) { + file.close(); + Util.rethrow(e); + } + throw ''; + } + + /** Returns absolute path, replacing `~` with homepath **/ + public static function getFullPath(path:String):String { + final splitPath = path.split("/"); + if (splitPath.length != 0 && splitPath.shift() == "~") { + return getHomePath() + "/" + splitPath.join("/"); + } + + return FileSystem.absolutePath(path); + } + + public static function zipDirectory(root:String):List { + final ret = new List(); + function seek(dir:String) { + for (name in FileSystem.readDirectory(dir)) + if (!name.startsWith('.')) { + final full = '$dir/$name'; + if (FileSystem.isDirectory(full)) + seek(full); + else { + final blob = File.getBytes(full); + final entry:Entry = { + fileName: full.substr(root.length + 1), + fileSize: blob.length, + fileTime: FileSystem.stat(full).mtime, + compressed: false, + dataSize: blob.length, + data: blob, + crc32: haxe.crypto.Crc32.make(blob), + }; + Tools.compress(entry, 9); + ret.push(entry); + } + } + } + seek(root); + return ret; + } } diff --git a/src/haxelib/api/GlobalScope.hx b/src/haxelib/api/GlobalScope.hx new file mode 100644 index 000000000..db166a979 --- /dev/null +++ b/src/haxelib/api/GlobalScope.hx @@ -0,0 +1,254 @@ +package haxelib.api; + +import sys.io.File; +import haxe.ds.GenericStack; +import haxe.io.Path; + +import haxelib.Data; + +import haxelib.api.LibraryData; +import haxelib.api.ScriptRunner; +import haxelib.api.Scope; +import haxelib.api.Hxml; + +using StringTools; + +/** + A Global Scope, which resolves libraries using the repository's global configured current + library versions. +**/ +class GlobalScope extends Scope { + function new(repository:Repository) { + super(false, repository); + } + + public function runScript(library:ProjectName, ?callData:CallData, ?version:Version):Void { + if (callData == null) + callData = {}; + final resolved = resolveVersionAndPath(library, version); + + final info = + try Data.readData(File.getContent(resolved.path + Data.JSON), false) + catch (e:Dynamic) + throw 'Failed when trying to parse haxelib.json for $library@${resolved.version}: $e'; + + // add dependency versions if given + final dependencies:Dependencies = + [for (name => version in info.dependencies) + Dependency.fromNameAndVersion(name, version)]; + + final libraryRunData:LibraryRunData = { + name: ProjectName.getCorrectOrAlias(info.name, library), + internalName: info.name, + version: resolved.version, + dependencies: dependencies, + path: resolved.path, + main: info.main + }; + + ScriptRunner.run(libraryRunData, resolveCompiler(), callData); + } + + public function getVersion(library:ProjectName):Version { + return repository.getCurrentVersion(library); + } + + public function setVersion(library:ProjectName, version:SemVer):Void { + repository.setCurrentVersion(library, version); + } + + public function setVcsVersion(library:ProjectName, vcsVersion:Vcs.VcsID, ?data:VcsData):Void { + if (data == null) data = {url: "unknown"}; + + if (data.subDir != null) { + final devDir = repository.getValidVersionPath(library, vcsVersion) + data.subDir; + repository.setDevPath(library, devDir); + } else { + repository.setCurrentVersion(library, vcsVersion); + } + } + + public function isLibraryInstalled(library:ProjectName):Bool { + return repository.isCurrentVersionSet(library); + } + + public function isOverridden(library:ProjectName):Bool { + if (!repository.isInstalled(library)) + return false; + return repository.getDevPath(library) != null; + } + + public function getPath(library:ProjectName, ?version:Version):String { + if (version != null) + return repository.getValidVersionPath(library, version); + + final devPath = repository.getDevPath(library); + if (devPath != null) + return devPath; + + final current = repository.getCurrentVersion(library); + return repository.getValidVersionPath(library, current); + } + + public function getLibraryNames():Array { + return repository.getLibraryNames(); + } + + public function getArrayOfLibraryInfo(?filter:String):Array { + final names = repository.getLibraryNames(filter); + + final projects = new Array(); + + for (name in names) { + final info = repository.getProjectInstallationInfo(name); + + projects.push({ + name: name, + current: try repository.getCurrentVersion(name) catch(e) null, + devPath: info.devPath, + versions: info.versions + }); + } + + return projects; + } + + public function getArgsAsHxml(library:ProjectName, ?version:Version):String { + final stack = new GenericStack(); + stack.add({library: library, version: version}); + + return getArgsAsHxmlWithDependencies(stack); + } + + public function getArgsAsHxmlForLibraries(libraries:Array<{library:ProjectName, version:Null}>):String { + final stack = new GenericStack<{library:ProjectName, version:Null}>(); + + for (i in 1...libraries.length + 1) + stack.add(libraries[libraries.length - i]); + + return getArgsAsHxmlWithDependencies(stack); + } + + function getArgsAsHxmlWithDependencies(stack:GenericStack<{library:ProjectName, version:Null}>){ + var argsString = ""; + function addLine(s:String) + argsString += '$s\n'; + + // the original set of inputs + final topLevelLibs = [for (lib in stack) lib]; + + final includedLibraries:Map = []; + + while (!stack.isEmpty()) { + final cur = stack.pop(); + // turn it to lowercase always (so that `LiBrArY:1.2.0` and `library:1.3.0` clash because + // they are different versions), and then get the correct name if provided + final library = repository.getCorrectName(cur.library); + final version = cur.version; + + // check for duplicates + if (includedLibraries.exists(library)) { + final otherVersion = includedLibraries[library]; + // if the current library is part of the original set of inputs, and if the versions don't match + if (topLevelLibs.contains(cur) && version != null && version != otherVersion) + throw 'Cannot process `${cur.library}:$version`: ' + + 'Library $library has two versions included : $otherVersion and $version'; + continue; + } + + final resolved = resolveVersionAndPath(library, version); + includedLibraries[library] = resolved.version; + + // neko libraries + final ndllDir = resolved.path + "ndll/"; + if (sys.FileSystem.exists(ndllDir)) + addLine('-L $ndllDir'); + + // extra parameters + try { + addLine(normalizeHxml(File.getContent(resolved.path + "extraParams.hxml"))); + } catch (_:Dynamic) {} + + final info = { + final jsonContent = try File.getContent(resolved.path + Data.JSON) catch (_) null; + Data.readData(jsonContent, jsonContent != null ? CheckSyntax : NoCheck, library); + } + + // path and version compiler define + addLine( + if (info.classPath != "") + Path.addTrailingSlash(Path.join([resolved.path, info.classPath])) + else + resolved.path + ); + // if info.name is not the placeholder for an empty name (i.e. unknown), use it, otherwise + // fall back to the value in the .name file + addLine('-D ${info.name}=${info.version}'); + + // add dependencies to stack + final dependencies = info.dependencies.toArray(); + + while (dependencies.length > 0) { + final dependency = dependencies.pop(); + stack.add({ + library: ProjectName.ofString(dependency.name), + version: // TODO: maybe check the git/hg commit hash here if it's given? + if (dependency.version == DependencyVersion.DEFAULT) null + else Version.ofString(dependency.version) + }); + } + } + + return argsString.trim(); + } + + static var haxeVersion(get, null):SemVer; + + static function get_haxeVersion():SemVer { + if (haxeVersion != null) + return haxeVersion; + + function attempt(cmd:String, arg:String, readStdErr = false):SemVer { + final p = new sys.io.Process(cmd, [arg]); + final outCode = p.exitCode(); + final err = p.stderr.readAll().toString(); + final versionStr = if (readStdErr) err else p.stdout.readAll().toString(); + p.close(); + if (outCode != 0) + throw 'Cannot get haxe version: $err'; + return SemVer.ofString(versionStr.split('+')[0]); + } + + return try { + // this works on haxe 4.0 and above + haxeVersion = attempt("haxe", "--version"); + } catch (_) { + // old haxe versions only understand `-version` + // they also print the version to stderr for whatever reason... + haxeVersion = attempt("haxe", "-version", true); + } + } + + function resolveCompiler():LibraryData { + return { + version: haxeVersion, + dependencies: [] + }; + } + + function resolveVersionAndPath(library:ProjectName, version:Null):{path:String, version:VersionOrDev} { + if (version != null) { + final path = repository.getValidVersionPath(library, version); + return {path: path, version: version}; + } + + final devPath = repository.getDevPath(library); + if (devPath != null) + return {path: devPath, version: Dev.Dev}; + + final current = repository.getCurrentVersion(library); + final path = repository.getValidVersionPath(library, current); + return {path: path, version: current}; + } + +} diff --git a/src/haxelib/api/Hxml.hx b/src/haxelib/api/Hxml.hx new file mode 100644 index 000000000..493409333 --- /dev/null +++ b/src/haxelib/api/Hxml.hx @@ -0,0 +1,15 @@ +package haxelib.api; + +using StringTools; + +private final regex = ~/\r?\n/g; + +/** + Normalizes `hxmlContents` by stripping comments, trimming whitespace + from each line and removing empty lines +**/ +function normalizeHxml(hxmlContents:String):String { + return regex.split(hxmlContents).map(StringTools.trim).filter(function(line) { + return line != "" && !line.startsWith("#"); + }).join('\n'); +} diff --git a/src/haxelib/api/Installer.hx b/src/haxelib/api/Installer.hx new file mode 100644 index 000000000..788efc664 --- /dev/null +++ b/src/haxelib/api/Installer.hx @@ -0,0 +1,862 @@ +package haxelib.api; + +import sys.FileSystem; +import sys.io.File; + +import haxelib.api.Repository; +import haxelib.api.Vcs; +import haxelib.api.LibraryData; +import haxelib.api.LibFlagData; + +using StringTools; +using Lambda; +using haxelib.Data; + +/** Exception thrown when an error occurs during installation. **/ +class InstallationException extends haxe.Exception {} +/** Exception thrown when a `vcs` error interrupts installation. **/ +class VcsCommandFailed extends InstallationException { + public final type:VcsID; + public final code:Int; + public final stdout:String; + public final stderr:String; + + public function new(type, code, stdout, stderr) { + this.type = type; + this.code = code; + this.stdout = stdout; + this.stderr = stderr; + super('$type command failed.'); + } +} + +/** Enum for indication the importance of a log message. **/ +enum LogPriority { + /** Regular messages **/ + Default; + /** Messages that can be ignored for cleaner output. **/ + Optional; + /** + Messages that are only useful for debugging purposes. Often for + raw executable output. + **/ + Debug; +} + +/** + A instance of a user interface used by an Installer instance. + + Contains functions that are executed on certain events or for + logging information. +**/ +@:structInit +class UserInterface { + final _log:Null<(msg:String, priority:LogPriority)-> Void>; + final _confirm:(msg:String)->Bool; + final _logDownloadProgress:Null<(filename:String, finished:Bool, cur:Int, max:Null, downloaded:Int, time:Float) -> Void>; + final _logInstallationProgress:Null<(msg:String, current:Int, total:Int) -> Void>; + + /** + `log` function used for logging information. + + `confirm` function used to confirm certain operations before they occur. + If it returns `true`, the operation will take place, + otherwise it will be cancelled. + + `downloadProgress` function used to track download progress of libraries from the Haxelib server. + + `installationProgress` function used to track installation progress. + **/ + public function new( + ?log:Null<(msg:String, priority:LogPriority)-> Void>, + ?confirm:Null<(msg:String)->Bool>, + ?logDownloadProgress:Null<(filename:String, finished:Bool, cur:Int, max:Null, downloaded:Int, time:Float) -> Void>, + ?logInstallationProgress:Null<(msg:String, current:Int, total:Int) -> Void> + ) { + _log = log; + _confirm = confirm != null ? confirm : (_)-> {true;}; + _logDownloadProgress = logDownloadProgress; + _logInstallationProgress = logInstallationProgress; + } + + public inline function log(msg:String, priority:LogPriority = Default):Void { + if (_log != null) + _log(msg, priority); + } + + public inline function confirm(msg:String):Bool { + return _confirm(msg); + } + + public inline function logInstallationProgress(msg:String, current:Int, total:Int):Void { + if (_logInstallationProgress != null) + _logInstallationProgress(msg, current, total); + } + + public inline function logDownloadProgress(filename:String, finished:Bool, cur:Int, max:Null, downloaded:Int, time:Float):Void { + if (_logDownloadProgress != null) + _logDownloadProgress(filename, finished, cur, max, downloaded, time); + } + + public inline function getDownloadProgressFunction():Null<(filename:String, finished:Bool, cur:Int, max:Null, downloaded:Int, time:Float) -> Void> { + return _logDownloadProgress; + } +} + +/** + Like `LibFlagData`, but `None` is not an option. + + Always contains enough information to reproduce + the same library installation. +**/ +enum InstallData { + Haxelib(version:SemVer); + VcsInstall(version:VcsID, vcsData:VcsData); +} + +private class AllInstallData { + public final name:ProjectName; + public final version:Version; + public final isLatest:Bool; + public final installData:InstallData; + + function new(name:ProjectName, version:Version, installData:InstallData, isLatest:Bool) { + this.name = name; + this.version = version; + this.installData = installData; + this.isLatest = isLatest; + } + + public static function create(name:ProjectName, libFlagData:LibFlagData, versionData:Null>):AllInstallData { + if (versionData != null && versionData.length == 0) + throw new InstallationException('The library $name has not yet released a version'); + + return switch libFlagData { + case None: + final semVer = getLatest(versionData); + new AllInstallData(name, semVer, Haxelib(semVer), true); + case Haxelib(version) if (!versionData.contains(version)): + throw new InstallationException('No such version $version for library $name'); + case Haxelib(version): + new AllInstallData(name, version, Haxelib(version), version == getLatest(versionData)); + case VcsInstall(version, vcsData): + new AllInstallData(name, version, VcsInstall(version, vcsData), false); + } + } +} + +private class AlreadyUpToDate extends InstallationException {} + +private function getLatest(versions:Array):SemVer { + if (versions.length == 0) + throw 'Library has not yet released a version'; + + final versions = versions.copy(); + versions.sort(function(a, b) return -SemVer.compare(a, b)); + + // get most recent non preview version + for (v in versions) + if (v.preview == null) + return v; + return versions[0]; // otherwise the most recent one +} + +/** Class for installing libraries into a scope and setting their versions. +**/ +class Installer { + /** If set to `true` library dependencies will not be installed. **/ + public static var skipDependencies = false; + + /** + If this is set to true, dependency versions will be reinstalled + even if already installed. + **/ + public static var forceInstallDependencies = false; + + final scope:Scope; + final repository:Repository; + final userInterface:UserInterface; + + final vcsBranchesByLibraryName = new Map(); + + /** + Creates a new Installer object that installs projects to `scope`. + + If `userInterface` is passed in, it will be used as the interface + for logging information and for operations that require user confirmation. + **/ + public function new(scope:Scope, ?userInterface:UserInterface){ + this.scope = scope; + repository = scope.repository; + + this.userInterface = userInterface != null? userInterface : {}; + } + + /** Installs library from the zip file at `path`. **/ + public function installLocal(path:String) { + final path = FileSystem.fullPath(path); + userInterface.log('Installing $path'); + // read zip content + final zip = FsUtils.unzip(path); + + final info = Data.readInfos(zip, false); + final library = info.name; + final version = info.version; + + installZip(info.name, info.version, zip); + + // set current version + scope.setVersion(library, version); + + userInterface.log(' Current version is now $version'); + userInterface.log("Done"); + + handleDependencies(library, version, info.dependencies); + } + + /** + Clears memory on git or hg library branches. + + An installer instance keeps track of updated vcs dependencies + to avoid cloning the same branch twice. + + This function can be used to clear that memory. + **/ + public function forgetVcsBranches():Void { + vcsBranchesByLibraryName.clear(); + } + + /** Installs libraries from the `haxelib.json` file at `path`. **/ + public function installFromHaxelibJson(path:String) { + final path = FileSystem.fullPath(path); + userInterface.log('Installing libraries from $path'); + + final dependencies = Data.readData(File.getContent(path), false).dependencies; + + try + installFromDependencies(dependencies) + catch (e) + throw new InstallationException('Failed installing dependencies from $path:\n$e'); + } + + /** + Installs the libraries required to build the HXML file at `path`. + + Throws an error when trying to install a library from the haxelib + server if the library has no versions or if the requested + version does not exist. + + If `confirmHxmlInstall` is passed in, it will be called with information + about the libraries to be installed, and the installation only proceeds if + it returns `true`. + **/ + public function installFromHxml(path:String, ?confirmHxmlInstall:(libs:Array<{name:ProjectName, version:Version}>) -> Bool) { + final path = FileSystem.fullPath(path); + userInterface.log('Installing all libraries from $path:'); + final libsToInstall = LibFlagData.fromHxml(path); + + if (libsToInstall.empty()) + return; + + // Check the version numbers are all good + userInterface.log("Loading info about the required libraries"); + + final installData = getFilteredInstallData(libsToInstall); + + final libVersions = [ + for (library in installData) + {name:library.name, version:library.version} + ]; + // Abort if not confirmed + if (confirmHxmlInstall != null && !confirmHxmlInstall(libVersions)) + return; + + for (library in installData) { + if (library.installData.match(Haxelib(_)) && repository.isVersionInstalled(library.name, library.version)) { + final version = SemVer.ofString(library.version); + if (scope.isLibraryInstalled(library.name) && scope.getVersion(library.name) == version) { + userInterface.log('Library ${library.name} version $version is already installed and set as current'); + } else { + userInterface.log('Library ${library.name} version $version is already installed'); + if (scope.isLocal || userInterface.confirm('Set ${library.name} to version $version')) { + scope.setVersion(library.name, version); + userInterface.log('Library ${library.name} current version is now $version'); + } + } + continue; + } + + try + installFromInstallData(library.name, library.installData) + catch (e) { + userInterface.log(e.toString()); + continue; + } + + final libraryName = switch library.installData { + case VcsInstall(version, vcsData): getVcsLibraryName(library.name, version, vcsData.subDir); + case _: library.name; + } + + setVersionAndLog(libraryName, library.installData); + + userInterface.log("Done"); + + handleDependenciesGeneral(libraryName, library.installData); + } + } + + /** + Installs `version` of `library` from the haxelib server. + + If `version` is omitted, the latest version is installed. + + If `forceSet` is set to true and the installer is running with + a global scope, the new version is always set as the current one. + + Otherwise, `version` is set as current if it is the latest version + of `library` or if no current version is set for `library`. + + If the `version` specified does not exist on the server, + an `InstallationException` is thrown. + **/ + public function installFromHaxelib(library:ProjectName, ?version:SemVer, forceSet:Bool = false) { + final info = Connection.getInfo(library); + // get correct name capitalization so that the logged output is correct + final library = ProjectName.ofString(info.name); + + final versions = [for (v in info.versions) v.name]; + + if (versions.length == 0) + throw new InstallationException('The library $library has not yet released a version'); + + final versionSpecified = version != null; + if (versionSpecified && !versions.contains(version)) + throw new InstallationException('No such version $version for library $library'); + + version = version ?? getLatest(versions); + + downloadAndInstall(library, version); + + // if no version was specified, we installed the latest version anyway + if (scope.isLocal || forceSet || !versionSpecified || version == getLatest(versions) || !scope.isLibraryInstalled(library)) { + scope.setVersion(library, version); + userInterface.log(' Current version is now $version'); + } + userInterface.log("Done"); + + handleDependencies(library, version); + } + + /** + Installs `library` from a git or hg repository (specified by `id`) + + `vcsData` contains information on the source repository + and the requested state. + **/ + public function installVcsLibrary(library:ProjectName, id:VcsID, vcsData:VcsData) { + installVcs(library, id, vcsData); + + library = getVcsLibraryName(library, id, vcsData.subDir); + + scope.setVcsVersion(library, id, vcsData); + + if (vcsData.subDir != null) { + final path = scope.getPath(library); + userInterface.log(' Development directory set to $path'); + } else { + userInterface.log(' Current version is now $id'); + } + userInterface.log("Done"); + + handleDependenciesVcs(library, id, vcsData.subDir); + } + + /** + Updates `library` to the newest version on the haxelib server, + or pull latest changes with git or hg. + **/ + public function update(library:ProjectName) { + try { + updateIfNeeded(library); + } catch (e:AlreadyUpToDate) { + userInterface.log(e.toString()); + } catch (e:VcsUpdateCancelled) { + return; + } + } + + /** + Updates all libraries in the scope. + + If a library update fails, it is skipped. + **/ + public function updateAll():Void { + final libraries = scope.getLibraryNames(); + var updated = false; + var failures = 0; + + for (library in libraries) { + userInterface.log('Checking $library'); + try { + updateIfNeeded(library); + updated = true; + } catch(e:AlreadyUpToDate) { + continue; + } catch (e:VcsUpdateCancelled) { + continue; + } catch(e) { + ++failures; + userInterface.log("Failed to update: " + e.toString()); + userInterface.log(e.stack.toString(), Debug); + } + } + + if (updated) { + if (failures == 0) { + userInterface.log("All libraries are now up-to-date"); + } else { + userInterface.log("All libraries are now up-to-date"); + } + } else + userInterface.log("All libraries are already up-to-date"); + } + + function updateIfNeeded(library:ProjectName) { + final current = try scope.getVersion(library) catch (_:CurrentVersionException) null; + + final vcsId = try VcsID.ofString(current) catch (_) null; + if (vcsId != null) { + final vcs = Vcs.get(vcsId); + if (vcs == null || !vcs.available) + throw 'Could not use $vcsId, please make sure it is installed and available in your PATH.'; + // with version locking we'll be able to be smarter with this + updateVcs(library, vcsId, vcs); + + scope.setVcsVersion(library, vcsId); + + handleDependenciesVcs(library, vcsId, null); + // we dont know if a subdirectory was given anymore + return; + } + + final semVer = try SemVer.ofString(current) catch (_) null; + + final info = Connection.getInfo(library); + final library = ProjectName.ofString(info.name); + final latest = info.getLatest(); + + if (semVer != null && semVer == latest) { + throw new AlreadyUpToDate('Library $library is already up to date'); + } else if (repository.isVersionInstalled(library, latest)) { + userInterface.log('Latest version $latest of $library is already installed'); + // only ask if running in a global scope + if (!scope.isLocal && !userInterface.confirm('Set $library to $latest')) + return; + } else { + downloadAndInstall(library, latest); + } + scope.setVersion(library, latest); + userInterface.log(' Current version is now $latest'); + userInterface.log("Done"); + + handleDependencies(library, latest); + } + + function getDependencies(path:String):Dependencies { + final jsonPath = path + Data.JSON; + if (!FileSystem.exists(jsonPath)) + return {}; + + return switch (Data.readData(File.getContent(jsonPath), false).dependencies) { + case null: {}; + case dependencies: dependencies; + } + } + + /** + Get the name found in the `haxelib.json` for a vcs library. + + If `givenName` is an alias (it is completely different from the internal name) + then `givenName` is returned instead + **/ + function getVcsLibraryName(givenName:ProjectName, id:VcsID, subDir:Null):ProjectName { + final jsonPath = scope.getPath(givenName, id) + (if (subDir != null) subDir else "") + Data.JSON; + if (!FileSystem.exists(jsonPath)) + return givenName; + final internalName = Data.readData(File.getContent(jsonPath), false).name; + return ProjectName.getCorrectOrAlias(internalName, givenName); + } + + function handleDependenciesGeneral(library:ProjectName, installData:InstallData) { + if (skipDependencies) + return; + + switch installData { + case Haxelib(version): + handleDependencies(library, version); + case VcsInstall(version, {subDir: subDir}): + handleDependenciesVcs(library, version, subDir); + } + } + + function handleDependenciesVcs(library:ProjectName, id:VcsID, subDir:Null) { + if (skipDependencies) + return; + + final path = repository.getVersionPath(library, id) + switch subDir { + case null: ''; + case subDir: subDir; + } + final dependencies = getDependencies(path); + + try + installFromDependencies(dependencies) + catch (e) + throw new InstallationException('Failed installing dependencies for $library:\n$e'); + } + + function handleDependencies(library:ProjectName, version:SemVer, dependencies:Dependencies = null) { + if (skipDependencies) + return; + + if (dependencies == null) + dependencies = getDependencies(repository.getVersionPath(library, version)); + + try + installFromDependencies(dependencies) + catch (e) + throw new InstallationException('Failed installing dependencies for $library:\n$e'); + } + + function installFromDependencies(dependencies:Dependencies) { + final libs = getLibFlagDataFromDependencies(dependencies); + + final installData = getInstallData(libs); + + for (lib in installData) { + final version = lib.version; + userInterface.log('Installing dependency ${lib.name} $version'); + + switch lib.installData { + case Haxelib(v) if (!forceInstallDependencies && repository.isVersionInstalled(lib.name, v)): + userInterface.log('Library ${lib.name} version $v is already installed'); + continue; + default: + } + + try + installFromInstallData(lib.name, lib.installData) + catch (e) { + userInterface.log(e.toString()); + continue; + } + + final library = switch lib.installData { + case VcsInstall(version, vcsData): getVcsLibraryName(lib.name, version, vcsData.subDir); + case _: lib.name; + } + + // vcs versions always get set + if (!scope.isLibraryInstalled(library) || lib.installData.match(VcsInstall(_))) { + setVersionAndLog(library, lib.installData); + } + + userInterface.log("Done"); + handleDependenciesGeneral(library, lib.installData); + } + } + + function getLibFlagDataFromDependencies(dependencies:Dependencies):List<{name:ProjectName, data:LibFlagData}> { + final list = new List<{name:ProjectName, data:LibFlagData}>(); + for (library => versionStr in dependencies) + // no version specified and dev set, no need to install dependency + if (forceInstallDependencies || !(versionStr == '' && scope.isOverridden(library))) + list.push({name: library, data: LibFlagData.extractFromDependencyString(versionStr)}); + + return list; + } + + function setVersionAndLog(library:ProjectName, installData:InstallData) { + switch installData { + case VcsInstall(version, vcsData): + scope.setVcsVersion(library, version, vcsData); + if (vcsData.subDir == null){ + userInterface.log(' Current version is now $version'); + } else { + final path = scope.getPath(library); + userInterface.log(' Development directory set to $path'); + } + case Haxelib(version): + scope.setVersion(library, version); + userInterface.log(' Current version is now $version'); + } + } + + static function getInstallData(libs:List<{name:ProjectName, data:LibFlagData}>):List { + final installData = new List(); + + final versionsData = getVersionsForEmptyLibs(libs); + + for (lib in libs) { + final data = versionsData[lib.name]; + if (data == null) { + installData.add(AllInstallData.create(lib.name, lib.data, null)); + continue; + } + final libName = data.confirmedName; + installData.add(AllInstallData.create(libName, lib.data, data.versions)); + } + + return installData; + } + + /** Returns a list of all require install data for the `libs`, and also filters out repeated libs. **/ + static function getFilteredInstallData(libs:List<{name:ProjectName, data:LibFlagData, isTargetLib:Bool}>):List { + final installData = new List(); + final includedLibs = new Map>(); + + final versionsData = getVersionsForEmptyLibs(libs); + + for (lib in libs) { + final allInstallData = { + final data = versionsData[lib.name]; + if (data == null) { + AllInstallData.create(lib.name, lib.data, null); + } else { + final libName = data.confirmedName; + AllInstallData.create(libName, lib.data, data.versions); + } + } + + final lowerCaseName = ProjectName.ofString(allInstallData.name.toLowerCase()); + + final includedVersions = includedLibs[lowerCaseName]; + if (includedVersions != null && (lib.isTargetLib || isVersionIncluded(allInstallData.installData, includedVersions))) + continue; // do not include twice + if (includedVersions == null) + includedLibs[lowerCaseName] = []; + includedLibs[lowerCaseName].push(allInstallData.installData); + installData.add(allInstallData); + } + + return installData; + } + + /** Returns a map of version information for libraries in `libs` that have empty version information. **/ + static function getVersionsForEmptyLibs(libs:List<{name:ProjectName, data:LibFlagData}>): + Map}> + { + final toCheck:Array = []; + for (lib in libs) + if (lib.data.match(None | Haxelib(_))) // Do not check vcs info + toCheck.push(lib.name); + + return Connection.getVersionsForLibraries(toCheck); + } + + static function isVersionIncluded(toCheck:InstallData, versions:Array):Bool { + for (version in versions) { + switch ([toCheck, version]) { + case [Haxelib(a), Haxelib(b)] if(a == b): return true; + case [VcsInstall(a, vcsData1), VcsInstall(b, vcsData2)] + if ((a == b) + && (vcsData1.url == vcsData2.url) + && (vcsData1.ref == vcsData2.ref) + && (vcsData1.branch == vcsData2.branch) + && (vcsData1.tag == vcsData2.tag) + && (vcsData1.subDir == vcsData2.subDir) + // maybe this equality check should be moved to vcsData + ): return true; + default: + } + } + return false; + } + + function installFromInstallData(library:ProjectName, data:InstallData) { + switch data { + case Haxelib(version): + downloadAndInstall(library, version); + case VcsInstall(version, vcsData): + installVcs(library, version, vcsData); + } + } + + function downloadAndInstall(library:ProjectName, version:SemVer) { + // download to temporary file + final filename = Data.fileName(library, version); + + userInterface.log('Downloading $filename...'); + + final filepath = haxe.io.Path.join([repository.path, filename]); + + final progressFunction = switch userInterface.getDownloadProgressFunction() { + case null: + null; + case fn: + (f, c, m, d, t) -> {fn('Downloading $filename', f, c, m, d, t);}; + }; + + Connection.download(filename, filepath, progressFunction); + + final zip = try FsUtils.unzip(filepath) catch (e) { + FileSystem.deleteFile(filepath); + Util.rethrow(e); + } + installZip(library, version, zip); + FileSystem.deleteFile(filepath); + + try + Connection.postInstall(library, version) + catch (e:Dynamic) {}; + } + + function installZip(library:ProjectName, version:SemVer, zip:List):Void { + userInterface.log('Installing $library...'); + + final versionPath = repository.getVersionPath(library, version); + FsUtils.safeDir(versionPath); + + // locate haxelib.json base path + final basepath = Data.locateBasePath(zip); + + // unzip content + final entries = [for (entry in zip) if (entry.fileName.startsWith(basepath)) entry]; + final total = entries.length; + for (i in 0...total) { + final zipfile = entries[i]; + final fileName = { + final full = zipfile.fileName; + // remove basepath + full.substr(basepath.length, full.length - basepath.length); + } + if (fileName.charAt(0) == "/" || fileName.charAt(0) == "\\" || fileName.split("..").length > 1) + throw new InstallationException("Invalid filename : " + fileName); + + userInterface.logInstallationProgress('Installing $library $version', i, total); + + final dirs = ~/[\/\\]/g.split(fileName); + final file = dirs.pop(); + + var path = ""; + for (d in dirs) { + path += d; + FsUtils.safeDir(versionPath + path); + path += "/"; + } + + if (file == "") { + if (path != "") + userInterface.log(' Created $path', Debug); + continue; // was just a directory + } + path += file; + userInterface.log(' Install $path', Debug); + File.saveBytes(versionPath + path, haxe.zip.Reader.unzip(zipfile)); + } + userInterface.logInstallationProgress('Done installing $library $version', total, total); + } + + function installVcs(library:ProjectName, id:VcsID, vcsData:VcsData) { + final vcs = Vcs.get(id); + if (vcs == null || !vcs.available) + throw 'Could not use $id, please make sure it is installed and available in your PATH.'; + + final libPath = repository.getVersionPath(library, id); + + final branch = vcsData.ref != null ? vcsData.ref : vcsData.branch; + final url:String = vcsData.url; + + function doVcsClone() { + userInterface.log('Installing $library from $url' + (branch != null ? " branch: " + branch : "")); + final tag = vcsData.tag; + try { + vcs.clone(libPath, url, branch, tag, userInterface.log.bind(_, Debug)); + } catch (error:VcsError) { + FsUtils.deleteRec(libPath); + switch (error) { + case VcsUnavailable(vcs): + throw 'Could not use ${vcs.executable}, please make sure it is installed and available in your PATH.'; + case CantCloneRepo(vcs, _, stderr): + throw 'Could not clone ${vcs.name} repository' + (stderr != null ? ":\n" + stderr : "."); + case CantCheckoutBranch(_, branch, stderr): + throw 'Could not checkout branch, tag or path "$branch": ' + stderr; + case CantCheckoutVersion(_, version, stderr): + throw 'Could not checkout tag "$version": ' + stderr; + case CommandFailed(_, code, stdout, stderr): + throw new VcsCommandFailed(id, code, stdout, stderr); + }; + } + } + + if (repository.isVersionInstalled(library, id)) { + userInterface.log('You already have $library version ${vcs.directory} installed.'); + + final wasUpdated = vcsBranchesByLibraryName.exists(library); + // difference between a key not having a value and the value being null + + final currentBranch = vcsBranchesByLibraryName[library]; + + // TODO check different urls as well + if (branch != null && (!wasUpdated || currentBranch != branch)) { + final currentBranchStr = currentBranch != null ? currentBranch : ""; + if (!userInterface.confirm('Overwrite branch: "$currentBranchStr" with "$branch"')) { + userInterface.log('Library $library $id repository remains at "$currentBranchStr"'); + return; + } + FsUtils.deleteRec(libPath); + doVcsClone(); + } else if (wasUpdated) { + userInterface.log('Library $library version ${vcs.directory} already up to date.'); + return; + } else { + userInterface.log('Updating $library version ${vcs.directory}...'); + try { + updateVcs(library, id, vcs); + } catch (e:AlreadyUpToDate){ + userInterface.log(e.toString()); + } + } + } else { + FsUtils.safeDir(libPath); + doVcsClone(); + } + + vcsBranchesByLibraryName[library] = branch; + } + + function updateVcs(library:ProjectName, id:VcsID, vcs:Vcs) { + final dir = repository.getVersionPath(library, id); + + final oldCwd = Sys.getCwd(); + Sys.setCwd(dir); + + final success = try { + vcs.update( + function() { + if (userInterface.confirm('Reset changes to $library $id repository in order to update to latest version')) + return true; + userInterface.log('$library repository has not been modified', Optional); + return false; + }, + userInterface.log.bind(_, Debug), + userInterface.log.bind(_, Optional) + ); + } catch (e:VcsError) { + Sys.setCwd(oldCwd); + switch e { + case CommandFailed(_, code, stdout, stderr): + throw new VcsCommandFailed(id, code, stdout, stderr); + default: Util.rethrow(e); // other errors aren't expected here + } + } catch (e:haxe.Exception) { + Sys.setCwd(oldCwd); + Util.rethrow(e); + } + Sys.setCwd(oldCwd); + if (!success) + throw new AlreadyUpToDate('Library $library $id repository is already up to date'); + userInterface.log('$library was updated'); + } +} diff --git a/src/haxelib/api/LibFlagData.hx b/src/haxelib/api/LibFlagData.hx new file mode 100644 index 000000000..d0988dc9e --- /dev/null +++ b/src/haxelib/api/LibFlagData.hx @@ -0,0 +1,147 @@ +package haxelib.api; + +import sys.io.File; + +import haxelib.Data; + +import haxelib.api.Hxml; +import haxelib.api.Vcs.VcsID; +import haxelib.api.LibraryData.VcsData; + +using StringTools; + +/** + Enum representing the different possible version information + that a Haxe `-lib` flag could hold, or a dependency string in a + `haxelib.json` file. + **/ +enum LibFlagData { + None; + Haxelib(version:SemVer); + VcsInstall(version:VcsID, vcsData:VcsData); +} + +private class LibParsingError extends haxe.Exception {} +private final libDataEReg = ~/^(.+?)(?::(.*))?$/; + +/** + Extracts library info from a full flag, + i.e.: `name:1.2.3` or `name:git:url#hash` +**/ +function extractFull(libFlag:String):{name:ProjectName, libFlagData:LibFlagData} { + if (!libDataEReg.match(libFlag)) + throw '$libFlag is not a valid library flag'; + + final name = ProjectName.ofString(libDataEReg.matched(1)); + final versionInfo = libDataEReg.matched(2); + + if (versionInfo == null) + return {name: name, libFlagData: None}; + + return {name: name, libFlagData: extractVersion(versionInfo)}; +} + +function extractFromDependencyString(str:DependencyVersion):LibFlagData { + if (str == "") + return None; + + return extractVersion(str); +} + +private function extractVersion(versionInfo:String):LibFlagData { + try { + return Haxelib(SemVer.ofString(versionInfo)); + } catch (_) {} + + try { + final data = getVcsData(versionInfo); + return VcsInstall(data.type, data.data); + } catch (e) { + throw '$versionInfo is not a valid library version'; + } +} + +private final vcsRegex = ~/^(git|hg)(?::(.+?)(?:#(?:([a-f0-9]{7,40})|(.+)))?)?$/; + +private function getVcsData(s:String):{type:VcsID, data:VcsData} { + if (!vcsRegex.match(s)) + throw '$s is not valid'; + final type = switch (vcsRegex.matched(1)) { + case Git: + Git; + case _: + Hg; + } + return { + type: type, + data: { + url: vcsRegex.matched(2), + ref: vcsRegex.matched(3), + branch: vcsRegex.matched(4), + subDir: null, + tag: null + } + } +} + +private final TARGETS = [ + "java" => ProjectName.ofString('hxjava'), + "jvm" => ProjectName.ofString('hxjava'), + "cpp" => ProjectName.ofString('hxcpp'), + "cs" => ProjectName.ofString('hxcs'), + "hl" => ProjectName.ofString('hashlink') +]; + +private final targetFlagEReg = { + final targetNameGroup = [for (target in TARGETS.keys()) target].join("|"); + new EReg('^--?($targetNameGroup) ', ""); +} + +private final libraryFlagEReg = ~/^-(lib|L|-library)\b/; + +/** + Extracts the lib information from the hxml file at `path`. + + Does not filter out repeated libs. + **/ +function fromHxml(path:String):List<{name:ProjectName, data:LibFlagData, isTargetLib:Bool}> { + final libsData = new List<{name:ProjectName, data:LibFlagData, isTargetLib:Bool}>(); + + final lines = [path]; + + while (lines.length > 0) { + final line = lines.shift().trim(); + if (line.endsWith(".hxml")) { + final newLines = normalizeHxml(File.getContent(line)).split("\n"); + newLines.reverse(); + for (line in newLines) + lines.unshift(line); + } + + // check targets + if (targetFlagEReg.match(line)) { + final target = targetFlagEReg.matched(1); + final lib = TARGETS[target]; + if (lib != null) + libsData.add({name: lib, data: None, isTargetLib: true}); + } + + if (libraryFlagEReg.match(line)) { + final key = libraryFlagEReg.matchedRight().trim(); + if (!libDataEReg.match(key)) + throw '$key is not a valid library flag'; + + final name = ProjectName.ofString(libDataEReg.matched(1)); + + libsData.add({ + name: name, + data: switch (libDataEReg.matched(2)) { + case null: None; + case v: extractVersion(v); + }, + isTargetLib: false + }); + } + } + return libsData; +} diff --git a/src/haxelib/api/LibraryData.hx b/src/haxelib/api/LibraryData.hx new file mode 100644 index 000000000..f4ab4b10f --- /dev/null +++ b/src/haxelib/api/LibraryData.hx @@ -0,0 +1,99 @@ +package haxelib.api; + +import haxe.DynamicAccess; + +import haxelib.api.Vcs.VcsID; + +/** Exception thrown upon errors regarding library data, such as invalid versions. **/ +class LibraryDataException extends haxe.Exception {} + +/** + Library version, which can be used in commands. + + This type of library version has a physical folder in the project root directory + (i.e. it is not a dev version) +**/ +abstract Version(String) to String from SemVer from VcsID { + inline function new(s:String) { + this = s; + } + + public static function ofString(s:String):Version { + if (!isValid(s)) + throw new LibraryDataException('`$s` is not a valid library version'); + return new Version(s); + } + + /** Returns whether `s` constitues a valid library version. **/ + public static function isValid(s:String):Bool { + return VcsID.isValid(s) || SemVer.isValid(s); + } +} + +/** A library version which can only be `dev`. **/ +@:noDoc +enum abstract Dev(String) to String { + final Dev = "dev"; +} + +/** Like `Version`, but also has the possible value of `dev`. **/ +abstract VersionOrDev(String) from VcsID from SemVer from Version from Dev to String {} + +/** Interface which all types of library data implement. **/ +interface ILibraryData { + final version:VersionOrDev; + final dependencies:Array; +} + +/** Data for a library installed from the haxelib server. **/ +@:structInit +class LibraryData implements ILibraryData { + public final version:SemVer; + public final dependencies:Array; +} + +/** Data for a library located in a local development path. **/ +@:structInit +class DevLibraryData implements ILibraryData { + public final version:Dev; + public final dependencies:Array; + public final path:String; +} + +/** Data for a library installed via vcs. **/ +@:structInit +class VcsLibraryData implements ILibraryData { + public final version:VcsID; + public final dependencies:Array; + /** Reproducible vcs information **/ + public final vcs:VcsData; +} + +private final hashRegex = ~/^([a-f0-9]{7,40})$/; +function isCommitHash(str:String) + return hashRegex.match(str); + +/** Class containing repoducible git or hg library data. **/ +@:structInit +class VcsData { + /** url from which to install **/ + public final url:String; + /** Commit hash **/ + @:optional + public final ref:Null; + /** The git tag or mercurial revision **/ + @:optional + public final tag:Null; + /** Branch **/ + @:optional + public final branch:Null; + /** + Sub directory in which the root of the project is found. + + Relative to project root + **/ + @:optional + public final subDir:Null; +} + +typedef LockFormat = DynamicAccess; diff --git a/src/haxelib/api/RepoManager.hx b/src/haxelib/api/RepoManager.hx new file mode 100644 index 000000000..7894a446f --- /dev/null +++ b/src/haxelib/api/RepoManager.hx @@ -0,0 +1,290 @@ +package haxelib.api; + +import sys.FileSystem; +import sys.io.File; + +import haxelib.api.FsUtils.*; + +using StringTools; +using haxe.io.Path; + +#if (haxe_ver < 4.1) +#error "RepoManager requires Haxe 4.1 or newer" +#end + +/** Exception thrown when an error happens when + changing or retrieving a repository path. + **/ +class RepoException extends haxe.Exception {} + +/** Enum representing the different possible causes + for a misconfigured global repository. + **/ +enum InvalidConfigurationType { + /** There is no configuration set **/ + NoneSet; + /** The configured folder does not exist **/ + NotFound(path:String); + /** The configuration points to a file instead of a directory **/ + IsFile(path:String); +} + +/** Exception thrown when a global repository has been misconfigured. +**/ +class InvalidConfiguration extends RepoException { + /** The type of configuration error. **/ + public final type:InvalidConfigurationType; + public function new(type:InvalidConfigurationType) { + final message = switch type { + case NoneSet: "No global repository has been configured"; + case NotFound(path): 'Haxelib repository $path does not exist'; + case IsFile(path): 'Haxelib repository $path exists, but is a file, not a directory'; + } + super(message); + this.type = type; + } +} + +/** Manager for the location of the haxelib database. **/ +class RepoManager { + static final REPO_DIR = "lib"; + static final LOCAL_REPO_DIR = ".haxelib"; + + static final CONFIG_FILE = ".haxelib"; + static final UNIX_SYSTEM_CONFIG_FILE = "/etc/.haxelib"; + + static final VARIABLE_NAME = "HAXELIB_PATH"; + + /** + Returns the path to the repository local to `dir` if one exists, + otherwise returns global repository path. + + If `dir` is omitted, the current working directory is used instead. + **/ + public static function getPath(?dir:String):String { + final dir = getDirectory(dir); + + final localPath = getLocalPath(dir); + if (localPath != null) + return localPath; + + return getValidGlobalPath(); + } + + /** + Searches for the path to local repository, starting in `dir` + and then going up until root directory is reached. + + Returns the directory path if it is found, otherwise returns null. + **/ + static function getLocalPath(dir:String):Null { + if (dir == "") + return null; + final repo = Path.join([dir, LOCAL_REPO_DIR]); + if (FileSystem.exists(repo) && FileSystem.isDirectory(repo)) + return FileSystem.fullPath(repo).addTrailingSlash(); + return getLocalPath(dir.directory()); + } + + /** + Returns the global repository path, but throws an exception + if it does not exist or if it is not a directory. + + The `HAXELIB_PATH` environment variable takes precedence over + the configured global repository path. + **/ + public static function getGlobalPath():String { + return getValidGlobalPath(); + } + + static function getValidGlobalPath():String { + final rep = readConfiguredGlobalPath(); + if (rep == null) { + if (!IS_WINDOWS) + throw new InvalidConfiguration(NoneSet); + // on Windows, we use the default one if none is set + final defaultPath = getDefaultGlobalPath(); + try + safeDir(defaultPath) + catch (e:Dynamic) + throw new RepoException('Error accessing Haxelib repository: $e'); + // configure the default as the global repository + File.saveContent(getConfigFilePath(), defaultPath); + initRepository(defaultPath); + return defaultPath; + } + if (!FileSystem.exists(rep)) + throw new InvalidConfiguration(NotFound(rep)); + else if (!FileSystem.isDirectory(rep)) + throw new InvalidConfiguration(IsFile(rep)); + + if (isRepositoryUninitialized(rep)) + initRepository(rep); + + return rep; + } + + /** + Sets `path` as the global haxelib repository in the user's haxelib config file. + + If `path` does not exist already, it is created. + **/ + public static function setGlobalPath(path:String):Void { + path = FileSystem.absolutePath(path); + final configFile = getConfigFilePath(); + + if (isSamePath(path, configFile)) + throw new RepoException('Cannot use $path because it is reserved for config file'); + + final isNew = safeDir(path); + if (isNew || FileSystem.readDirectory(path).length == 0) // if we created the path or if it is empty + initRepository(path); + + File.saveContent(configFile, path); + } + + /** + Deletes the user's current haxelib setup, + resetting their global repository path. + **/ + public static function unsetGlobalPath():Void { + final configFile = getConfigFilePath(); + FileSystem.deleteFile(configFile); + } + + /** + Returns the previous global repository path if a valid one had been + set up, otherwise returns the default path for the current operating + system. + **/ + public static function suggestGlobalPath() { + final configured = readConfiguredGlobalPath(); + if (configured != null) + return configured; + + return getDefaultGlobalPath(); + } + + /** + Returns the global Haxelib repository path, without validating + that it exists. If it is not configured anywhere, returns `null`. + + First checks `HAXELIB_PATH` environment variable, + then checks the content of user config file. + + On Unix-like systems also checks `/etc/.haxelib` for system wide + configuration. + **/ + static function readConfiguredGlobalPath():Null { + // first check the env var + final environmentVar = Sys.getEnv(VARIABLE_NAME); + if (environmentVar != null) + return environmentVar.trim().addTrailingSlash(); + + // try to read from user config + try { + return getTrimmedContent(getConfigFilePath()).addTrailingSlash(); + } catch (_) { + // the code below could go in here instead, but that + // results in extra nesting... + } + + if (!IS_WINDOWS) { + // on unixes, try to read system-wide config + /* TODO the system wide config has never been configured in haxelib code. + Either configure it somewhere or remove this bit of code? */ + try { + return getTrimmedContent(UNIX_SYSTEM_CONFIG_FILE).addTrailingSlash(); + } catch (_) {} + } + return null; + } + + /** + Creates a new local repository in the directory `dir` if one doesn't already exist. + + If `dir` is ommited, the current working directory is used. + + Throws RepoException if repository already exists. + **/ + public static function createLocal(?dir:String) { + if (! (dir == null || FileSystem.exists(dir))) + FsUtils.safeDir(dir); + final dir = getDirectory(dir); + final path = FileSystem.absolutePath(Path.join([dir, LOCAL_REPO_DIR])); + final created = FsUtils.safeDir(path, true); + if(!created) + throw new RepoException('Local repository already exists ($path)'); + initRepository(path); + } + + /** + Deletes the local repository in the directory `dir`, if it exists. + + If `dir` is ommited, the current working directory is used. + + Throws RepoException if no repository is found. + **/ + public static function deleteLocal(?dir:String) { + final dir = getDirectory(dir); + final path = FileSystem.absolutePath(Path.join([dir, LOCAL_REPO_DIR])); + final deleted = FsUtils.deleteRec(path); + if (!deleted) + throw new RepoException('No local repository found ($path)'); + } + + static function isRepositoryUninitialized(path:String) { + return FileSystem.readDirectory(path).length == 0; + } + + static function initRepository(path:String) { + RepoReformatter.initRepoVersion(path); + } + + static function getConfigFilePath():String { + return Path.join([getHomePath(), CONFIG_FILE]); + } + + /** Returns the default path for the global directory. **/ + static function getDefaultGlobalPath():String { + if (IS_WINDOWS) + return getWindowsDefaultGlobalPath(); + + // TODO `lib/` is for binaries, see if we can move all of these to `share/` + return if (FileSystem.exists("/usr/share/haxe/")) // for Debian + '/usr/share/haxe/$REPO_DIR/' + else if (Sys.systemName() == "Mac") // for newer OSX, where /usr/lib is not writable + '/usr/local/lib/haxe/$REPO_DIR/' + else '/usr/lib/haxe/$REPO_DIR/'; // for other unixes + } + + /** + The Windows haxe installer will setup `%HAXEPATH%`. + We will default haxelib repo to `%HAXEPATH%/lib.` + + When there is no `%HAXEPATH%`, we will use a `/haxelib` + directory next to the config file, ".haxelib". + **/ + static function getWindowsDefaultGlobalPath():String { + final haxepath = Sys.getEnv("HAXEPATH"); + if (haxepath != null) + return Path.join([haxepath.trim(), REPO_DIR]).addTrailingSlash(); + return Path.join([getConfigFilePath().directory(), "haxelib"]).addTrailingSlash(); + } + + static function getTrimmedContent(filePath:String):String { + return File.getContent(filePath).trim(); + } + + static function getDirectory(dir:Null):String { + if (dir == null) + return Sys.getCwd(); + return Path.addTrailingSlash( + try { + FileSystem.fullPath(dir); + } catch (e) { + throw '$dir does not exist'; + } + ); + } +} diff --git a/src/haxelib/api/RepoReformatter.hx b/src/haxelib/api/RepoReformatter.hx new file mode 100644 index 000000000..b9c8ce696 --- /dev/null +++ b/src/haxelib/api/RepoReformatter.hx @@ -0,0 +1,201 @@ +package haxelib.api; + +import haxe.ds.Either; +import sys.FileSystem; +import sys.io.File; +import haxe.io.Path; + +import haxelib.ProjectName; +import haxelib.api.LibraryData; + +using Lambda; +using StringTools; + +/** + Responsible for checking if a Repository requires reformatting + as well as carrying out the reformatting. +**/ +class RepoReformatter { + /** To increment whenever the repository format changes. **/ + static final CURRENT_REPO_VERSION = 1; + + static final REPO_VERSION_FILE = ".repo-version"; + + + /** + Returns true if the repository version is lower + than the version supported by the current version of this library. + **/ + public static function doesRepositoryRequireReformat(repo:Repository):Bool { + return getRepositoryVersion(repo) < CURRENT_REPO_VERSION; + } + + /** + Returns true if the repository version is higher than + the version supported by the current version of this library. + **/ + public static function isRepositoryIncompatible(repo:Repository):Bool { + return getRepositoryVersion(repo) > CURRENT_REPO_VERSION; + } + + @:allow(haxelib.api.RepoManager) + static function initRepoVersion(path:String):Void { + setRepoVersion(path, CURRENT_REPO_VERSION); + } + + static function setRepoVersion(path:String, version:Int):Void { + File.saveContent(Path.join([path, REPO_VERSION_FILE]), Std.string(version) + "\n"); + } + + static function getRepositoryVersion(repo:Repository):Int { + return try { + Std.parseInt(File.getContent(Path.join([repo.path, REPO_VERSION_FILE])).trim()); + } catch (e) { + 0; + } + } + + /** + Reformats `repo` to the current version supported by this + version of the library. + + `log` can be passed in for logging information. + + If `repo`'s version is equal to the version supported, then it is + treated as if we are updating from the first ever version. + If `repo`'s version is greater than what is supported by + this version of the library, an exception is thrown. + **/ + public static function reformat(repo:Repository, ?log:(msg:String)-> Void):Void { + if (log == null) log = (_)-> {}; + + final version = + switch getRepositoryVersion(repo) { + case version if (version < CURRENT_REPO_VERSION): + version; + case version if (version == CURRENT_REPO_VERSION): + 0; + case version: + throw 'Repository has version $version, but this library only supports up to $CURRENT_REPO_VERSION.\n' + + 'Reformatting cannot be done.'; + } + + for (v => update in updaterFunctions.slice(version)){ + log('Updating from version $v to ${v+1}'); + update(repo, log); + setRepoVersion(repo.path, v + 1); + } + log('Repository is now at version: $CURRENT_REPO_VERSION'); + } + + // updater functions should check for issues before making any changes. + static final updaterFunctions = [ + updateFrom0 + ]; + + /** Updates from version 0 to 1. **/ + static function updateFrom0(repo:Repository, log:(msg:String)->Void) { + final path = repo.path; + + final filteredItems = []; + + // map of new paths and the old paths that will be moved to them + final newPaths:Map>> = []; + + if (!FsUtils.IS_WINDOWS) log("Checking for conflicts"); + + for (subDir in FileSystem.readDirectory(path)) { + final oldPath = Path.join([path, subDir]); + final lowerCaseVersion = subDir.toLowerCase(); + if (subDir.startsWith(".") || !FileSystem.isDirectory(oldPath) || lowerCaseVersion == subDir) + continue; + + filteredItems.push(subDir); + + if (FsUtils.IS_WINDOWS) continue; + final newPath = Path.join([repo.path, lowerCaseVersion]); + switch newPaths[newPath] { + case null if (!FileSystem.exists(newPath)): + newPaths[newPath] = Left(oldPath); + case null: + newPaths[newPath] = Right([newPath, oldPath]); + case Left(single): + newPaths[newPath] = Right([single, oldPath]); + case Right(arr): + arr.push(oldPath); + } + } + + // look for potential conflicts + for (_ => item in newPaths) { + final items = switch item { + case Left(_): continue; // only one folder, so there are no conflicts + case Right(arr): arr; + } + + final pathByItem:Map = []; + + for (oldPath in items) { + for (subDir in FileSystem.readDirectory(oldPath)) { + final lower = subDir.toLowerCase(); + final existing = pathByItem[lower]; + final fullPath = Path.join([oldPath, subDir]); + + if (existing == null) { // no conflict + pathByItem[lower] = fullPath; + continue; + } + // conflict!!! + final message = switch (lower) { + case ".name": + continue; + case ".dev": + 'There are two conflicting dev versions set:'; + case ".current": + 'There are two conflicting current versions set:'; + case other if (Version.isValid(Data.unsafe(other))): + 'There are two conflicting versions in:'; + case _: + 'There are two conflicting unrecognized files/folders:'; + }; + throw '$message\n`$existing` and `$fullPath`\nPlease remove one manually and retry.'; + } + } + } + + if (!FsUtils.IS_WINDOWS) + filteredItems.sort(Reflect.compare); + + for (subDir in filteredItems) { + final fullPath = Path.join([repo.path, subDir]); + final subDirLower = subDir.toLowerCase(); + final newPath = Path.join([repo.path, subDirLower]); + + try { + log('Moving `$fullPath` to `$newPath`'); + FileSystem.rename(fullPath, newPath); + } catch(_){ + for (subItem in FileSystem.readDirectory(fullPath)) { + final itemPath = '$fullPath/$subItem'; + final newItemPath = '$newPath/$subItem'; + if (FileSystem.exists(newItemPath)){ + FileSystem.deleteFile(itemPath); + continue; // must have been cleared + } + log('Moving `$itemPath` to `$newItemPath`'); + FileSystem.rename(itemPath, newItemPath); + } + log('Deleting `$fullPath`'); + FileSystem.deleteDirectory(fullPath); + } + + final library = ProjectName.ofString(Data.unsafe(subDir)); + final nameFile = Path.join([newPath, @:privateAccess Repository.NAME_FILE]); + + if (!FileSystem.exists(nameFile)) { + log('Setting name for `$library`'); + File.saveContent(nameFile, subDir); + } + } + } +} diff --git a/src/haxelib/api/Repository.hx b/src/haxelib/api/Repository.hx new file mode 100644 index 000000000..3769ef3f8 --- /dev/null +++ b/src/haxelib/api/Repository.hx @@ -0,0 +1,470 @@ +package haxelib.api; + +import sys.FileSystem; +import sys.io.File; + +import haxelib.api.RepoManager; +import haxelib.api.LibraryData; + +using Lambda; +using StringTools; +using haxe.io.Path; + +/** + Exception thrown when there is an error with the configured + current version of a library. + **/ +class CurrentVersionException extends haxe.Exception {} + +/** + Instance of a repository which can be used to get information on + library versions installed in the repository, as well as + directly modifying them. + **/ +class Repository { + /** Name of file used to keep track of current version. **/ + static final CURRENT_FILE = ".current"; + + /** Name of file used to keep track of capitalization. **/ + static final NAME_FILE = ".name"; + + /** + The path to the repository. + + If this field is being accessed and the repository path + is missing, an exception is thrown. + **/ + public var path(get, null):String; + + function get_path() { + if (!FileSystem.exists(path)) + throw new RepoException('Repository at $path no longer exists.'); + return path; + } + + function new(path:String) { + this.path = path; + } + /** + Returns a Repository instance for the local repository + if one is found, otherwise for the global one. + + If `dir` is omitted, the current working directory is used instead. + **/ + public static function get(?dir:String):Repository { + return new Repository(RepoManager.getPath(dir)); + } + + /** + Returns a Repository instance for the global repository. + **/ + public static function getGlobal():Repository { + return new Repository(RepoManager.getGlobalPath()); + } + + /** + Returns an array of installed project names. + + If `filter` is given, ignores projects that do not + contain it as a substring. + **/ + public function getLibraryNames(filter:String = null):Array { + if (filter != null) + filter = filter.toLowerCase(); + + inline function isFilteredOut(name:String) { + if (filter == null) + return false; + return !name.contains(filter); + } + + final projects = []; + + for (dir in FileSystem.readDirectory(path)) { + // hidden, not a folder, or has upper case letters + if (dir.startsWith(".") || dir.toLowerCase() != dir || !FileSystem.isDirectory(Path.join([path, dir]))) + continue; + + final allLower = try ProjectName.ofString(Data.unsafe(dir)) catch (_) continue; + final libraryName = getCapitalization(allLower); + + if (!isFilteredOut(allLower)) + projects.push(libraryName); + } + return projects; + } + + /** Returns information on currently installed versions for project `name` **/ + public function getProjectInstallationInfo(name:ProjectName):{versions: Array, devPath:String} { + final semVers:Array = []; + final others:Array = []; + final root = getProjectRootPath(name); + + for (sub in FileSystem.readDirectory(root)) { + // ignore .dev and .current files + if (sub.startsWith(".")) + continue; + + final version = Data.unsafe(sub); + try { + final semVer = SemVer.ofString(version); + semVers.push(semVer); + } catch(e:haxe.Exception) { + if (Vcs.VcsID.isValid(version)) + others.push(Vcs.VcsID.ofString(version)); + } + } + if (semVers.length != 0) + semVers.sort(SemVer.compare); + + final versions = (semVers:Array).concat(others); + + return { + versions: versions, + devPath: getDevPath(name) + }; + } + + /** Returns whether project `name` is installed **/ + public function isInstalled(name:ProjectName):Bool { + return FileSystem.exists(getProjectRootPath(name)); + } + + /** Returns whether `version` of project `name` is installed **/ + public function isVersionInstalled(name:ProjectName, version:Version):Bool { + return FileSystem.exists(getProjectVersionPath(name, version)); + } + + /** + Removes the project `name` from the repository. + Throws an error if `name` is not installed. + **/ + public function removeProject(name:ProjectName) { + final path = getProjectRootPath(name); + + if (!FileSystem.exists(path)) + throw 'Library $name is not installed'; + + FsUtils.deleteRec(path); + } + + /** + Removes `version` of project `name`. + + Throws an exception if: + + - `name` or `version` is not installed + - `version` matches the current version + - the project's development path is set as the path of `version` + - the project's development path is set to a subdirectory of it. + **/ + public function removeProjectVersion(name:ProjectName, version:Version) { + if (!FileSystem.exists(getProjectRootPath(name))) + throw 'Library $name is not installed'; + + final versionPath = getProjectVersionPath(name, version); + if (!FileSystem.exists(versionPath)) + throw 'Library $name version $version is not installed'; + + final current = getCurrentFileContent(name); + if (current == version) + throw 'Cannot remove current version of library $name'; + + try { + confirmRemovalAgainstDev(name, versionPath); + } catch (e) { + throw 'Cannot remove library `$name` version `$version`: $e\n' + + 'Use `haxelib dev $name` to unset the dev path'; + } + + FsUtils.deleteRec(versionPath); + } + + /** Throws an error if removing `versionPath` conflicts with the dev path of library `name` **/ + function confirmRemovalAgainstDev(name:ProjectName, versionPath:String) { + final devFilePath = getDevFilePath(name); + if (!FileSystem.exists(devFilePath)) + return; + + final devPath = filterAndNormalizeDevPath(File.getContent(devFilePath).trim()); + + if (devPath.startsWith(versionPath)) + throw 'It holds the `dev` version of `$name`'; + } + + /** + Set current version of project `name` to `version`. + + Throws an error if `name` or `version` of `name` is not installed. + + `name` may also be used as the official version of the library name + if installing a proper semver release + **/ + public function setCurrentVersion(name:ProjectName, version:Version):Void { + if (!FileSystem.exists(getProjectRootPath(name))) + throw 'Library $name is not installed'; + + if (!FileSystem.exists(getProjectVersionPath(name, version))) + throw 'Library $name version $version is not installed'; + + final currentFilePath = getCurrentFilePath(name); + final isNewLibrary = !FileSystem.exists(currentFilePath); + + File.saveContent(currentFilePath, version); + + // if the library is being installed for the first time, or this is a proper version + // or it is a git/hg/dev version but there are no proper versions installed + // proper semver releases replace names given by git/hg/dev versions + if (isNewLibrary || SemVer.isValid(version) || !doesLibraryHaveOfficialVersion(name)) + setCapitalization(name); + } + + /** + Returns the current version of project `name`. + **/ + public function getCurrentVersion(name:ProjectName):Version { + if (!FileSystem.exists(getProjectRootPath(name))) + throw new CurrentVersionException('Library $name is not installed'); + + final content = getCurrentFileContent(name); + return try + Version.ofString(content) + catch (e:LibraryDataException) + throw new CurrentVersionException('Current set version of $name is invalid.'); + } + + /** + Returns whether project `name` has a valid current version set. + **/ + public function isCurrentVersionSet(name:ProjectName):Bool { + if (!FileSystem.exists(getProjectRootPath(name))) + return false; + + final content = try + getCurrentFileContent(name) + catch(_:CurrentVersionException) + return false; + + return Version.isValid(content); + } + + /** + Returns the path for `version` of project `name`, + throwing an error if the project or version is not installed. + **/ + public function getValidVersionPath(name:ProjectName, version:Version):String { + if (!FileSystem.exists(getProjectRootPath(name))) + throw 'Library $name is not installed'; + + final path = getProjectVersionPath(name, version); + if (!FileSystem.exists(path)) + throw 'Library $name version $version is not installed'; + + return path; + } + + /** + Returns the root path project `name`, + without confirming that it is installed. + **/ + public function getProjectPath(name:ProjectName):String { + return getProjectRootPath(name); + } + + /** + Returns the path for `version` of project `name`, + without confirming that the project or version are installed. + **/ + public function getVersionPath(name:ProjectName, version:Version):String { + return getProjectVersionPath(name, version); + } + + /** + Returns the correctly capitalized name for library `name`. + + `name` can be any possible capitalization variation of the library name. + **/ + public function getCorrectName(name:ProjectName):ProjectName { + final rootPath = getProjectRootPath(name); + if (!FileSystem.exists(rootPath)) + throw 'Library $name is not installed'; + + return getCapitalization(name); + } + + static inline function doesNameHaveCapitals(name:ProjectName):Bool { + return name != name.toLowerCase(); + } + + function setCapitalization(name:ProjectName):Void { + // if it is not all lowercase then we save the actual capitalisation in the `.name` file + final filePath = addToRepoPath(name, NAME_FILE); + + if (doesNameHaveCapitals(name)) { + File.saveContent(filePath, name); + return; + } + + if (FileSystem.exists(filePath)) + FileSystem.deleteFile(filePath); + } + + function getCapitalization(name:ProjectName):ProjectName { + final filePath = addToRepoPath(name, NAME_FILE); + if (!FileSystem.exists(filePath)) + return name.toLowerCase(); + + final content = try { + File.getContent(filePath); + } catch (e) { + throw 'Failed when checking the name for library \'$name\': $e'; + } + + return ProjectName.ofString(content.trim()); + } + + // returns whether or not `name` has any versions installed which are not dev/git/hg + function doesLibraryHaveOfficialVersion(name:ProjectName):Bool { + final root = getProjectRootPath(name); + + for (sub in FileSystem.readDirectory(root)) { + // ignore .dev and .current files + if (sub.startsWith(".")) + continue; + + final version = Data.unsafe(sub); + if (SemVer.isValid(version)) + return true; + } + return false; + } + + inline function getCurrentFilePath(name:ProjectName):String { + return addToRepoPath(name, CURRENT_FILE); + } + + inline function getCurrentFileContent(name:ProjectName):String { + final currentFile = getCurrentFilePath(name); + if (!FileSystem.exists(currentFile)) + throw new CurrentVersionException('No current version set for library \'$name\''); + + try + return File.getContent(currentFile).trim() + catch (e) + throw new CurrentVersionException('Failed when reading the current version for library \'$name\': $e'); + } + + inline function getProjectRootPath(name:ProjectName):String { + return addToRepoPath(name).addTrailingSlash(); + } + + inline function getProjectVersionPath(name:ProjectName, version:Version):String { + final versionDir:String = + switch version { + case v if (SemVer.isValid(v)): v; + case (try Vcs.VcsID.ofString(_) catch(_) null) => vcs if (vcs != null): + Vcs.getDirectoryFor(vcs); + case _: throw 'Unknown library version: $version'; // we shouldn't get here + } + return addToRepoPath(name, Data.safe(versionDir).toLowerCase()).addTrailingSlash(); + } + + inline function addToRepoPath(name:ProjectName, ?sub:String):String { + return Path.join([ + path, + Data.safe(name).toLowerCase(), + if (sub != null) + sub + else + "" + ]); + } + + // Not sure about these: + // https://github.com/HaxeFoundation/haxe/wiki/Haxe-haxec-haxelib-plan#legacy-haxelib-features + + static final DEV_FILE = ".dev"; + /** + Sets the dev path for project `name` to `path`. + **/ + public function setDevPath(name:ProjectName, path:String) { + final root = getProjectRootPath(name); + + final isNew = !FileSystem.exists(root); + + FileSystem.createDirectory(root); + + if (isNew || !doesLibraryHaveOfficialVersion(name)) { + setCapitalization(name); + } + + final devFile = Path.join([root, DEV_FILE]); + + File.saveContent(devFile, normalizeDevPath(path)); + } + + /** + Removes the development directory for `name`, if one was set. + **/ + public function removeDevPath(name:ProjectName) { + final devFile = getDevFilePath(name); + if (FileSystem.exists(devFile)) + FileSystem.deleteFile(devFile); + } + + /** + Returns the development path for `name`. + If no development path is set, or it is filtered out, + returns null. + **/ + public function getDevPath(name:ProjectName):Null { + if (!FileSystem.exists(getProjectRootPath(name))) + throw 'Library $name is not installed'; + + final devFile = getDevFilePath(name); + if (!FileSystem.exists(devFile)) + return null; + + return filterAndNormalizeDevPath(File.getContent(devFile).trim()); + } + + function filterAndNormalizeDevPath(devPath:String):Null { + final path = normalizeDevPath(devPath); + + if (isDevPathExcluded(path)) + return null; + + return path; + } + + static function normalizeDevPath(devPath:String):Null { + // windows environment variables + final expanded = ~/%([A-Za-z0-9_]+)%/g.map( + devPath, + function(r) { + final env = Sys.getEnv(r.matched(1)); + return env == null ? "" : env; + }); + + return Path.normalize(expanded).addTrailingSlash(); + } + + static function isDevPathExcluded(normalizedPath:String):Bool { + final filters = switch (Sys.getEnv("HAXELIB_DEV_FILTER")) { + case null: // no filters set + return false; + case filterStr: + filterStr.split(";"); + } + + // check that `path` does not start with any of the filtered paths + return !filters.exists(function(flt) { + final normalizedFilter = Path.normalize(flt).toLowerCase(); + return normalizedPath.toLowerCase().startsWith(normalizedFilter); + }); + } + + function getDevFilePath(name:ProjectName):String { + return addToRepoPath(name, DEV_FILE); + } +} diff --git a/src/haxelib/api/Scope.hx b/src/haxelib/api/Scope.hx new file mode 100644 index 000000000..4897b099b --- /dev/null +++ b/src/haxelib/api/Scope.hx @@ -0,0 +1,134 @@ +package haxelib.api; + +import haxelib.api.LibraryData; +import haxelib.api.ScriptRunner; + +using StringTools; + +/** Information on the installed versions of a library. **/ +typedef InstallationInfo = { + final name:ProjectName; + final versions:Array; + final current:String; + final devPath:Null; +} + +/** + Returns scope for directory `dir`. If `dir` is omitted, uses the current + working directory. + + The scope will resolve libraries to the local repository if one exists, + otherwise to the global one. +**/ +function getScope(?dir:String):Scope { + if (dir == null) + dir = Sys.getCwd(); + @:privateAccess + return new GlobalScope(Repository.get(dir)); +} + +/** Returns the global scope. **/ +function getGlobalScope(?dir:String):GlobalScope { + @:privateAccess + return new GlobalScope(Repository.get(dir)); +} + +/** + Returns scope created for directory `dir`, resolving libraries to `repository` + + If `dir` is omitted, uses the current working directory. +**/ +function getScopeForRepository(repository:Repository, ?dir:String):Scope { + if (dir == null) + dir = Sys.getCwd(); + @:privateAccess + return new GlobalScope(repository); +} + +/** + This is an abstract class which the GlobalScope (and later on LocalScope) + inherits from. + + It is responsible for managing current library versions, resolving them, + giving information on them, or running them. +**/ +abstract class Scope { + /** Whether the scope is local. **/ + public final isLocal:Bool; + /** The repository which is used to resolve the scope's libraries. **/ + public final repository:Repository; + final overrides:LockFormat; + + function new(isLocal:Bool, repository:Repository) { + this.isLocal = isLocal; + this.repository = repository; + + overrides = loadOverrides(); + } + + /** + Runs the script for `library` with `callData`. + + If `version` is specified, that version will be used, + or an error is thrown if that version isn't installed + in the scope. + + Should the script return a non zero code, a ScriptError + exception is thrown containing the error code. + **/ + public abstract function runScript(library:ProjectName, ?callData:CallData, ?version:Version):Void; + + /** Returns the current version of `library`, ignoring overrides and dev directories. **/ + public abstract function getVersion(library:ProjectName):Version; + + /** + Set `library` to `version`. + + Requires that the library is already installed. + **/ + public abstract function setVersion(library:ProjectName, version:SemVer):Void; + /** + Set `library` to `vcsVersion`, with `data`. + + If `data` is omitted or incomplete then the required data is obtained manually. + + Requires that the library is already installed. + **/ + public abstract function setVcsVersion(library:ProjectName, vcsVersion:Vcs.VcsID, ?data:VcsData):Void; + + /** Returns whether `library` is currently installed in this scope (ignoring overrides). **/ + public abstract function isLibraryInstalled(library:ProjectName):Bool; + + /** Returns whether `library` version is currently overridden. **/ + public abstract function isOverridden(library:ProjectName):Bool; + + /** Returns an array of the libraries in the scope. **/ + public abstract function getLibraryNames():Array; + + /** + Returns an array of installation information on libraries in the scope. + + If `filter` is given, ignores libraries that do not contain it as a substring. + **/ + public abstract function getArrayOfLibraryInfo(?filter:String):Array; + + /** Returns the path to the source directory of `version` of `library`. **/ + public abstract function getPath(library:ProjectName, ?version:Version):String; + + /** Returns the required build arguments for `version` of `library` as an hxml string. **/ + public abstract function getArgsAsHxml(library:ProjectName, ?version:Version):String; + + /** + Returns the required build arguments for each library version in `libraries` + as one combined hxml string. + **/ + public abstract function getArgsAsHxmlForLibraries(libraries:Array<{library:ProjectName, version:Null}>):String; + + abstract function resolveCompiler():LibraryData; + + // TODO: placeholders until https://github.com/HaxeFoundation/haxe/wiki/Haxe-haxec-haxelib-plan + static function loadOverrides():LockFormat { + return {}; + } + +} diff --git a/src/haxelib/api/ScriptRunner.hx b/src/haxelib/api/ScriptRunner.hx new file mode 100644 index 000000000..9ca538782 --- /dev/null +++ b/src/haxelib/api/ScriptRunner.hx @@ -0,0 +1,180 @@ +package haxelib.api; + +import sys.FileSystem; + +import haxelib.Data.DependencyVersion; +import haxelib.api.LibraryData; + +/** Contains data with which a library script is executed. **/ +@:structInit +class CallData { + /** Directory in which the script will start. Defaults to current working directory. **/ + public final dir = Sys.getCwd(); + /** Array of arguments to be passed on to the library call. **/ + public final args:Array = []; + /** + Whether to pass `--haxelib-global` to the haxe compiler when running + (only works with haxe 4.0.0 and above). + **/ + public final useGlobalRepo:Bool = false; +} + +@:noDoc +/** either the project `name` or `name:version` **/ +abstract Dependency(String) from ProjectName to String { + inline function new(d) this = d; + public static function fromNameAndVersion(name:ProjectName, version:DependencyVersion):Dependency + return new Dependency(switch cast(version, String) { + case '': name; + case version: '$name:$version'; + }); +} + +@:noDoc +typedef Dependencies = Array; + +@:noDoc +/** Library data needed in order to run it **/ +typedef LibraryRunData = { + /** This may be an alias. **/ + final name:ProjectName; + /** This is the actual name found in the `haxelib.json`. **/ + final internalName:ProjectName; + final version:VersionOrDev; + final dependencies:Dependencies; + final main:Null; + final path:String; +} + +private enum RunType { + /** Runs compiled neko file at `path` **/ + Neko(path:String); + /** **/ + Script(main:String, name:ProjectName, version:VersionOrDev, dependencies:Dependencies); +} + +private typedef State = { + final dir:String; + final run:Null; + final runName:Null; +} + +/** Exception which is thrown if running a library script returns a non-zero code. **/ +@:noDoc +class ScriptError extends haxe.Exception { + /** The error code returned by the library script call. **/ + public final code:Int; + public function new(name:ProjectName, code:Int) { + super('Script for library "$name" exited with error code: $code'); + this.code = code; + } +} + +/** Class containing function for running a library's script. **/ +@:noDoc +class ScriptRunner { + static final HAXELIB_RUN = "HAXELIB_RUN"; + static final HAXELIB_RUN_NAME = "HAXELIB_RUN_NAME"; + + /** + Run `library`, with `callData`. + + `compilerData` is used if it is an interpreted script. + **/ + public static function run(library:LibraryRunData, compilerData:LibraryData, callData:CallData):Void { + final type = getType(library); + + final cmd = getCmd(type); + final args = generateArgs(type, callData, SemVer.ofString(compilerData.version)); + + final oldState = getState(); + + // call setup + setState({ + dir: library.path, + run: "1", + runName: library.internalName + }); + + final output = Sys.command(cmd, args); + + // return to previous state + setState(oldState); + + if (output != 0) + throw new ScriptError(library.name, output); + } + + static function getType(library:LibraryRunData) { + if (library.main != null) + return Script(library.main, library.name, library.version, library.dependencies); + if (FileSystem.exists(library.path + 'run.n')) + return Neko(library.path + 'run.n'); + if (FileSystem.exists(library.path + 'Run.hx')) + return Script("Run", library.name, library.version, library.dependencies); + throw 'Library ${library.name} version ${library.version} does not have a run script'; + } + + static function getCmd(runType:RunType):String { + return switch runType { + case Neko(_): "neko"; + case Script(_): "haxe"; + } + } + + static function generateArgs(runType:RunType, callData:CallData, compilerVersion:SemVer):Array { + switch runType { + case Neko(path): + final callArgs = callData.args.copy(); + callArgs.unshift(path); + callArgs.push(callData.dir); + return callArgs; + case Script(main, name, version, dependencies): + final isHaxe4 = SemVer.compare(compilerVersion, SemVer.ofString('4.0.0')) >= 0; + final useGlobalRepo = isHaxe4 && callData.useGlobalRepo; + + final callArgs = generateScriptArgs(main, name, version, dependencies, useGlobalRepo); + for (arg in callData.args) + callArgs.push(arg); + callArgs.push(callData.dir); + return callArgs; + } + } + + static function generateScriptArgs(main:String, name:ProjectName, version:VersionOrDev, dependencies:Dependencies, useGlobalRepo:Bool):Array { + final args = []; + + function addLib(data:String):Void { + args.push("--library"); + args.push(data); + } + + if (useGlobalRepo) + args.push('--haxelib-global'); + + // add the project itself first + addLib(if (version != Dev.Dev) '$name:$version' else '$name'); + + for (d in dependencies) + addLib(d); + + args.push('--run'); + args.push(main); + return args; + } + + static function getState():State { + return { + dir: Sys.getCwd(), + run: Sys.getEnv(HAXELIB_RUN), + runName: Sys.getEnv(HAXELIB_RUN_NAME) + }; + } + + static function setState(state:State):Void { + Sys.setCwd(state.dir); + Sys.putEnv(HAXELIB_RUN, state.run); + Sys.putEnv(HAXELIB_RUN_NAME, state.runName); + } + +} diff --git a/src/haxelib/client/Vcs.hx b/src/haxelib/api/Vcs.hx similarity index 52% rename from src/haxelib/client/Vcs.hx rename to src/haxelib/api/Vcs.hx index 37bc31ed4..4eb84a8f2 100644 --- a/src/haxelib/client/Vcs.hx +++ b/src/haxelib/api/Vcs.hx @@ -19,21 +19,20 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ -package haxelib.client; +package haxelib.api; import sys.FileSystem; -using haxelib.client.Vcs; +using haxelib.api.Vcs; interface IVcs { /** The name of the vcs system. **/ - var name(default, null):String; + final name:String; /** The directory used to install vcs library versions to. **/ - var directory(default, null):String; + final directory:String; /** The vcs executable. **/ - var executable(default, null):String; + final executable:String; /** Whether or not the executable can be accessed successfully. **/ var available(get, null):Bool; - var settings(default, null):Settings; /** Clone repository at `vcsPath` into `libPath`. @@ -41,21 +40,47 @@ interface IVcs { If `branch` is specified, the repository is checked out to that branch. `version` can also be specified for tags in git or revisions in mercurial. + + `debugLog` will be used to log executable output. **/ - function clone(libPath:String, vcsPath:String, ?branch:String, ?version:String):Void; + function clone(libPath:String, vcsPath:String, ?branch:String, ?version:String, ?debugLog:(msg:String)->Void):Void; /** Updates repository in CWD or CWD/`Vcs.directory` to HEAD. For git CWD must be in the format "...haxelib-repo/lib/git". + + By default, uncommitted changes prevent updating. + If `confirm` is passed in, the changes may occur + if `confirm` returns true. + + `debugLog` will be used to log executable output. + + `summaryLog` may be used to log summaries of changes. + Returns `true` if update successful. **/ - function update(libName:String):Bool; + function update(?confirm:()->Bool, ?debugLog:(msg:String)->Void, ?summaryLog:(msg:String)->Void):Bool; } /** Abstract enum representing the types of Vcs systems that are supported. **/ @:enum abstract VcsID(String) to String { final Hg = "hg"; final Git = "git"; + + /** Returns `true` if `s` constitutes a valid VcsID **/ + public static function isValid(s:String) { + return s == Hg || s == Git; + } + + /** Returns `s` as a VcsID if it is valid, otherwise throws an error. **/ + public static function ofString(s:String):VcsID { + if (s == Git) + return Git; + else if (s == Hg) + return Hg; + else + throw 'Invalid VscID $s'; + } } /** Enum representing errors that can be thrown during a vcs operation. **/ @@ -64,72 +89,60 @@ enum VcsError { CantCloneRepo(vcs:Vcs, repo:String, ?stderr:String); CantCheckoutBranch(vcs:Vcs, branch:String, stderr:String); CantCheckoutVersion(vcs:Vcs, version:String, stderr:String); + CommandFailed(vcs:Vcs, code:Int, stdout:String, stderr:String); } - -typedef Settings = { - @:optional final flat:Bool; - @:optional final debug:Bool; - @:optional final quiet:Bool; -} - +/** Exception thrown when a vcs update is cancelled. **/ +class VcsUpdateCancelled extends haxe.Exception {} /** Base implementation of `IVcs` for `Git` and `Mercurial` to extend. **/ -class Vcs implements IVcs { - static var reg:Map; - - public var name(default, null):String; - public var directory(default, null):String; - public var executable(default, null):String; - public var settings(default, null):Settings; +abstract class Vcs implements IVcs { + /** If set to true, recursive cloning is disabled **/ + public static var flat = false; + public final name:String; + public final directory:String; + public final executable:String; public var available(get, null):Bool; var availabilityChecked = false; var executableSearched = false; - public static function initialize(settings:Settings) { - if (reg == null) { - reg = [ - VcsID.Git => new Git(settings), - VcsID.Hg => new Mercurial(settings) - ]; - } else { - if (reg.get(VcsID.Git) == null) - reg.set(VcsID.Git, new Git(settings)); - if (reg.get(VcsID.Hg) == null) - reg.set(VcsID.Hg, new Mercurial(settings)); - } - } - - - function new(executable:String, directory:String, name:String, settings:Settings) { + function new(executable:String, directory:String, name:String) { this.name = name; this.directory = directory; this.executable = executable; - this.settings = { - flat: settings.flat != null ? settings.flat : false, - debug: settings.debug != null ? settings.debug : false, - quiet: settings.quiet != null && !settings.debug ? settings.quiet : false - } } + static var reg:Map; + + /** Returns the Vcs instance for `id`. **/ + public static function get(id:VcsID):Null { + if (reg == null) + reg = [ + VcsID.Git => new Git("git", "git", "Git"), + VcsID.Hg => new Mercurial("hg", "hg", "Mercurial") + ]; - public static function get(id:VcsID, settings:Settings):Null { - initialize(settings); return reg.get(id); } - static function set(id:VcsID, vcs:Vcs, settings:Settings, ?rewrite:Bool):Void { - initialize(settings); + /** Returns the sub directory to use for library versions of `id`. **/ + public static function getDirectoryFor(id:VcsID):String { + return switch (get(id)) { + case null: throw 'Unable to get directory for $id'; + case vcs: vcs.directory; + } + } + + static function set(id:VcsID, vcs:Vcs, ?rewrite:Bool):Void { final existing = reg.get(id) != null; if (!existing || rewrite) reg.set(id, vcs); } /** Returns the relevant Vcs if a vcs version is installed at `libPath`. **/ - public static function getVcsForDevLib(libPath:String, settings:Settings):Null { - initialize(settings); + public static function getVcsForDevLib(libPath:String):Null { for (k in reg.keys()) { if (FileSystem.exists(libPath + "/" + k) && FileSystem.isDirectory(libPath + "/" + k)) return reg.get(k); @@ -137,49 +150,12 @@ class Vcs implements IVcs { return null; } - function sure(commandResult:{code:Int, out:String}):Void { - switch (commandResult) { - case {code: 0}: //pass - case {code: code, out:out}: - if (!settings.quiet) - Sys.stderr().writeString(out); - Sys.exit(code); - } - } - - function command(cmd:String, args:Array):{ - code: Int, - out: String - } { - final p = try { - new sys.io.Process(cmd, args); - } catch(e:Dynamic) { - return { - code: -1, - out: Std.string(e) - } - } - final out = p.stdout.readAll().toString(); - final err = p.stderr.readAll().toString(); - if (settings.debug && out != "") - Sys.println(out); - if (settings.debug && err != "") - Sys.stderr().writeString(err); - final code = p.exitCode(); - final ret = { - code: code, - out: code == 0 ? out : err - }; - p.close(); - return ret; - } - function searchExecutable():Void { executableSearched = true; } function checkExecutable():Bool { - available = (executable != null) && try command(executable, []).code == 0 catch(_:Dynamic) false; + available = (executable != null) && try run([]).code == 0 catch(_:Dynamic) false; availabilityChecked = true; if (!available && !executableSearched) @@ -188,30 +164,76 @@ class Vcs implements IVcs { return available; } - @:final function get_available():Bool { + final function get_available():Bool { if (!availabilityChecked) checkExecutable(); return available; } - public function clone(libPath:String, vcsPath:String, ?branch:String, ?version:String):Void { - throw "This method must be overridden."; + final function run(args:Array, ?debugLog:(msg:String) -> Void, strict = false):{ + code:Int, + out:String, + err:String, + } { + inline function print(msg) + if (debugLog != null && msg != "") + debugLog(msg); + + print("# Running command: " + executable + " " + args.toString() + "\n"); + + final proc = command(executable, args); + if (strict && proc.code != 0) + throw CommandFailed(this, proc.code, proc.out, proc.err); + + print(proc.out); + print(proc.err); + print('# Exited with code ${proc.code}\n'); + + return proc; } - public function update(libName:String):Bool { - throw "This method must be overridden."; + static function command(cmd:String, args:Array):{ + code:Int, + out:String, + err:String, + } { + final p = try { + new sys.io.Process(cmd, args); + } catch (e:Dynamic) { + return { + code: -1, + out: "", + err: Std.string(e) + } + } + final out = p.stdout.readAll().toString(); + final err = p.stderr.readAll().toString(); + final code = p.exitCode(); + final ret = { + code: code, + out: out, + err: err + }; + p.close(); + return ret; } + + public abstract function clone(libPath:String, vcsPath:String, ?branch:String, ?version:String, ?debugLog:(msg:String)->Void):Void; + + public abstract function update(?confirm:() -> Bool, ?debugLog:(msg:String) -> Void, ?summaryLog:(msg:String) -> Void):Bool; } /** Class wrapping `git` operations. **/ class Git extends Vcs { - public function new(settings:Settings) - super("git", "git", "Git", settings); + @:allow(haxelib.api.Vcs.get) + function new(executable:String, directory:String, name:String) { + super(executable, directory, name); + } override function checkExecutable():Bool { // with `help` cmd because without any cmd `git` can return exit-code = 1. - available = (executable != null) && try command(executable, ["help"]).code == 0 catch(_:Dynamic) false; + available = (executable != null) && try run(["help"]).code == 0 catch(_:Dynamic) false; availabilityChecked = true; if (!available && !executableSearched) @@ -248,27 +270,29 @@ class Git extends Vcs { } } - override public function update(libName:String):Bool { + public function update(?confirm:()->Bool, ?debugLog:(msg:String)->Void, ?_):Bool { if ( - command(executable, ["diff", "--exit-code", "--no-ext-diff"]).code != 0 - || - command(executable, ["diff", "--cached", "--exit-code", "--no-ext-diff"]).code != 0 + run(["diff", "--exit-code", "--no-ext-diff"], debugLog).code != 0 + || run(["diff", "--cached", "--exit-code", "--no-ext-diff"], debugLog).code != 0 ) { - if (Cli.ask("Reset changes to " + libName + " " + name + " repo so we can pull latest version")) { - sure(command(executable, ["reset", "--hard"])); - } else { - if (!settings.quiet) - Sys.println(name + " repo left untouched"); - return false; - } + if (confirm == null || !confirm()) + throw new VcsUpdateCancelled('$name update in ${Sys.getCwd()} was cancelled'); + run(["reset", "--hard"], debugLog, true); } - final code = command(executable, ["pull"]).code; + run(["fetch"], debugLog, true); + + // `git rev-parse @{u}` will fail if detached + final checkUpstream = run(["rev-parse", "@{u}"], debugLog); + + if (checkUpstream.out == run(["rev-parse", "HEAD"], debugLog, true).out) + return false; // already up to date + // But if before we pulled specified branch/tag/rev => then possibly currently we haxe "HEAD detached at ..". - if (code != 0) { + if (checkUpstream.code != 0) { // get parent-branch: final branch = { - final raw = command(executable, ["show-branch"]).out; + final raw = run(["show-branch"], debugLog).out; final regx = ~/\[([^]]*)\]/; if (regx.match(raw)) regx.matched(1); @@ -276,36 +300,37 @@ class Git extends Vcs { raw; } - sure(command(executable, ["checkout", branch, "--force"])); - sure(command(executable, ["pull"])); + run(["checkout", branch, "--force"], debugLog, true); } + run(["merge"], debugLog, true); return true; } - override public function clone(libPath:String, url:String, ?branch:String, ?version:String):Void { + public function clone(libPath:String, url:String, ?branch:String, ?version:String, ?debugLog:(msg:String)->Void):Void { final oldCwd = Sys.getCwd(); final vcsArgs = ["clone", url, libPath]; - if (settings == null || !settings.flat) + if (!Vcs.flat) vcsArgs.push('--recursive'); - //TODO: move to Vcs.run(vcsArgs) - //TODO: use settings.quiet - if (command(executable, vcsArgs).code != 0) + if (run(vcsArgs, debugLog).code != 0) throw VcsError.CantCloneRepo(this, url/*, ret.out*/); - Sys.setCwd(libPath); if (version != null && version != "") { - final ret = command(executable, ["checkout", "tags/" + version]); - if (ret.code != 0) + final ret = run(["checkout", "tags/" + version], debugLog); + if (ret.code != 0) { + Sys.setCwd(oldCwd); throw VcsError.CantCheckoutVersion(this, version, ret.out); + } } else if (branch != null) { - final ret = command(executable, ["checkout", branch]); - if (ret.code != 0) + final ret = run(["checkout", branch], debugLog); + if (ret.code != 0){ + Sys.setCwd(oldCwd); throw VcsError.CantCheckoutBranch(this, branch, ret.out); + } } // return prev. cwd: @@ -316,8 +341,10 @@ class Git extends Vcs { /** Class wrapping `hg` operations. **/ class Mercurial extends Vcs { - public function new(settings:Settings) - super("hg", "hg", "Mercurial", settings); + @:allow(haxelib.api.Vcs.get) + function new(executable:String, directory:String, name:String) { + super(executable, directory, name); + } override function searchExecutable():Void { super.searchExecutable(); @@ -336,41 +363,37 @@ class Mercurial extends Vcs { checkExecutable(); } - override public function update(libName:String):Bool { - command(executable, ["pull"]); - var summary = command(executable, ["summary"]).out; - final diff = command(executable, ["diff", "-U", "2", "--git", "--subrepos"]); - final status = command(executable, ["status"]); + public function update(?confirm:()->Bool, ?debugLog:(msg:String)->Void, ?summaryLog:(msg:String)->Void):Bool { + inline function log(msg:String) if(summaryLog != null) summaryLog(msg); + + run(["pull"], debugLog); + var summary = run(["summary"], debugLog).out; + final diff = run(["diff", "-U", "2", "--git", "--subrepos"], debugLog); + final status = run(["status"], debugLog); // get new pulled changesets: // (and search num of sets) summary = summary.substr(0, summary.length - 1); summary = summary.substr(summary.lastIndexOf("\n") + 1); // we don't know any about locale then taking only Digit-exising:s - var changed = ~/(\d)/.match(summary); - if (changed && !settings.quiet) + final changed = ~/(\d)/.match(summary); + if (changed) // print new pulled changesets: - Sys.println(summary); - + log(summary); if (diff.code + status.code + diff.out.length + status.out.length != 0) { - if (!settings.quiet) - Sys.println(diff.out); - if (Cli.ask("Reset changes to " + libName + " " + name + " repo so we can update to latest version")) { - sure(command(executable, ["update", "--clean"])); - } else { - changed = false; - if (!settings.quiet) - Sys.println(name + " repo left untouched"); - } + log(diff.out); + if (confirm == null || !confirm()) + throw new VcsUpdateCancelled('$name update in ${Sys.getCwd()} was cancelled'); + run(["update", "--clean"], debugLog, true); } else if (changed) { - sure(command(executable, ["update"])); + run(["update"], debugLog, true); } return changed; } - override public function clone(libPath:String, url:String, ?branch:String, ?version:String):Void { + public function clone(libPath:String, url:String, ?branch:String, ?version:String, ?debugLog:(msg:String)->Void):Void { final vcsArgs = ["clone", url, libPath]; if (branch != null) { @@ -383,7 +406,7 @@ class Mercurial extends Vcs { vcsArgs.push(version); } - if (command(executable, vcsArgs).code != 0) + if (run(vcsArgs, debugLog).code != 0) throw VcsError.CantCloneRepo(this, url/*, ret.out*/); } } diff --git a/src/haxelib/client/Args.hx b/src/haxelib/client/Args.hx new file mode 100644 index 000000000..ba865deda --- /dev/null +++ b/src/haxelib/client/Args.hx @@ -0,0 +1,310 @@ +package haxelib.client; + +using StringTools; + +class SwitchError extends haxe.Exception {} +class InvalidCommand extends haxe.Exception {} + +@:structInit +class ArgsInfo { + public final command:Command; + public final mainArgs: Array; + public final flags: Array; + public final options: Map; + public final repeatedOptions: Map>; +} + +enum CommandCategory { + Basic; + Information; + Development; + Miscellaneous; +} + +enum abstract Command(String) to String { + final Install = "install"; + final Update = "update"; + final Remove = "remove"; + final List = "list"; + final Set = "set"; + + final Search = "search"; + final Info = "info"; + final User = "user"; + final Config = "config"; + final Path = "path"; + final LibPath = "libpath"; + final Version = "version"; + final Help = "help"; + + final Submit = "submit"; + final Register = "register"; + final Dev = "dev"; + final Git = "git"; + final Hg = "hg"; + + final Setup = "setup"; + final NewRepo = "newrepo"; + final DeleteRepo = "deleterepo"; + final FixRepo = "fixrepo"; + final ConvertXml = "convertxml"; + final Run = "run"; + final Proxy = "proxy"; + // deprecated commands + final Local = "local"; + final SelfUpdate = "selfupdate"; + + static final ALIASES = [ + "upgrade" => Update + ]; + + static final COMMANDS = Util.getValues(Command); + + public static function ofString(str:String):Null { + // first check aliases + final alias = ALIASES.get(str); + if(alias != null) + return alias; + // then check the rest + for (command in COMMANDS) { + if (cast(command, String) == str) + return command; + } + return null; + } + +} + +enum abstract Flag(String) to String { + final Global = "global"; + final Debug = "debug"; + final Quiet = "quiet"; + final Flat = "flat"; + final Always = "always"; + final Never = "never"; + final System = "system"; + final SkipDependencies = "skip-dependencies"; + // hidden + final NoTimeout = "notimeout"; + + public static final MUTUALLY_EXCLUSIVE = [[Quiet, Debug], [Always, Never]]; + + /** + Priority flags that need to be accessed prior to + complete argument parsing. + **/ + public static final PRIORITY = [System, Debug, Global]; + + static final FLAGS = Util.getValues(Flag); + + public static function ofString(str:String):Null { + for (flag in FLAGS) { + if ((flag:String) == str) + return flag; + } + return null; + } + +} + +enum abstract Option(String) to String { + final Remote = "R"; + + static final OPTIONS = Util.getValues(Option); + + public static function ofString(str:String):Null