diff --git a/Manifest.toml b/Manifest.toml index 1337aac..9c04b2e 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -121,6 +121,12 @@ version = "0.9.18" [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +[[deps.Formatting]] +deps = ["Printf"] +git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8" +uuid = "59287772-0a20-5a39-b81b-1366585eb4c0" +version = "0.4.2" + [[deps.Future]] deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" diff --git a/Project.toml b/Project.toml index afae8f7..bc9c71a 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "1.9.0" [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" Debugger = "31a5f54b-26ea-5ae9-a837-f05ce5417438" +Formatting = "59287772-0a20-5a39-b81b-1366585eb4c0" GraphQLClient = "09d831e3-9c21-47a9-bfd8-076871817219" JuliaFormatter = "98e50ef6-434e-11e9-1051-2b60c6c9e899" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" diff --git a/docs/src/index.md b/docs/src/index.md index b0c1b0c..9305265 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -20,7 +20,7 @@ pkg> add https://github.com/graphprotocol/AllocationOpt.jl/ pkg> add Comonicon ``` -## Public Function +## Public API We expose the following functions in our public API. diff --git a/docs/src/usage.md b/docs/src/usage.md index 06be347..0a0f517 100644 --- a/docs/src/usage.md +++ b/docs/src/usage.md @@ -1,11 +1,37 @@ # Usage -## Configuring The Indexer Service +## ActionQueue Set indexer-service flag `--serve-network-subgraph` to `true` so optimiser can pull data from the network subgraph at the indexer-service's network endpoint. If you would like to keep it false, you can instead supply an API url to `indexer_service_network_url`. -## CLI Usage +1. Download the [*allocationopt* script](https://raw.githubusercontent.com/graphprotocol/AllocationOpt.jl/main/scripts/allocationopt). For example, using `curl` or `wget`. Make sure you use the raw file! +2. Make the *allocationopt* script executable. + +```bash +$ chmod +x allocationopt +``` + +3. Run the *allocationopt* script with the *actionqueue* option. If this doesn't work, check that the shebang in the allocationopt file points to your julia executable. + +```bash +$ ./scripts/allocationopt --help +$ ./scripts/allocationopt actionqueue "0x6ac85b9d834b51b14a7b0ed849bb5199e04c05c5" test/example.csv 0.0 10 http://localhost:18000 http://localhost:7600/network +``` + +3. On Linux, you can further simplify the use of this script by symlinking it to your *.local/bin* + +``` +$ mkdir -p ~/.local/bin +$ cd ~/.local/bin +$ ln -s ~/projects/AllocationOpt.jl/scripts/allocationopt . # Change to the path to the allocationopt script for you +``` + +You should now be able to run the *allocationopt* script from anywhere! Similarly to how you'd run *ls*. + +## Indexer Rules + +If you don't have the action queue set up yet, you can also run the optimiser by telling it to generate indexing rules. 1. Download the [*allocationopt* script](https://raw.githubusercontent.com/graphprotocol/AllocationOpt.jl/main/scripts/allocationopt). For example, using `curl` or `wget`. Make sure you use the raw file! 2. Make the *allocationopt* script executable. @@ -14,11 +40,11 @@ If you would like to keep it false, you can instead supply an API url to `indexe $ chmod +x allocationopt ``` -3. Run the *allocationopt* script. If this doesn't work, check that the shebang in the allocationopt file points to your julia executable. +3. Run the *allocationopt* script with the *rules* option. If this doesn't work, check that the shebang in the allocationopt file points to your julia executable. ```bash $ ./scripts/allocationopt --help -$ ./scripts/allocationopt "0x6ac85b9d834b51b14a7b0ed849bb5199e04c05c5" test/example.csv 0.0 10 http://localhost:18000 http://localhost:7600/network +$ ./scripts/allocationopt rules "0x6ac85b9d834b51b14a7b0ed849bb5199e04c05c5" test/example.csv 0.0 10 http://localhost:18000 ``` 3. On Linux, you can further simplify the use of this script by symlinking it to your *.local/bin* diff --git a/scripts/allocationopt b/scripts/allocationopt index 89109d5..6e76498 100755 --- a/scripts/allocationopt +++ b/scripts/allocationopt @@ -3,7 +3,7 @@ using Comonicon using AllocationOpt """ -Optimises an indexer's allocations. +Optimises an indexer's allocations and pushes them to the action queue. # Arguments @@ -15,7 +15,7 @@ Optimises an indexer's allocations. - `management_server_url`: The URL that exposes the indexer managment server, including the port. Must begin with http. Example: http://localhost:18000. - `indexer_service_network_url`: The URL that exposes the indexer service's network endpoint. Must begin with http. Example: http://localhost:7600/network. """ -@main function main(id, filepath, minimum_allocation_amount, maximum_new_allocations, management_server_url, indexer_service_network_url) +@cast function actionqueue(id, filepath, minimum_allocation_amount, maximum_new_allocations, management_server_url, indexer_service_network_url) # Read subgraph lists defined in the file cols = read_filterlists(filepath) @@ -26,9 +26,41 @@ Optimises an indexer's allocations. ω = optimize_indexer(repo, indexer, parse(Float64, minimum_allocation_amount), parse(Int64, maximum_new_allocations)) # Push results to action queue - response = push_allocations!(id, management_server_url, indexer_service_network_url, ω, cols...) + _ = push_allocations!(id, management_server_url, indexer_service_network_url, ω, cols...) println("Done!") - return ω + return nothing end + +""" +Optimises an indexer's allocations and generates indexer rules to change allocations. + +# Arguments + +- `id`: The id of the indexer to optimise. +- `filepath`: A path to the CSV file that contains whitelist, blacklist, pinnedlist, frozenlist as columns. +- `grtgas`: The maximum amount of GRT that you are willing to spend on each allocation transaction. +- `minimum_allocation_amount`: The minimum amount of GRT that you are willing to allocate to a subgraph. +- `allocation_lifetime`: The number of epoches you assume to open each allocations for. Lifetime 1 means an epoch should be closed in its second epoch +- `indexer_service_network_url`: The URL that exposes the indexer service's network endpoint. Must begin with http. Example: http://localhost:7600/network. +""" +@cast function rules(id, filepath, minimum_allocation_amount, maximum_new_allocations, indexer_service_network_url) + # Read subgraph lists defined in the file + cols = read_filterlists(filepath) + + # Pull network state from indexer service network endpoint + indexer, repo = network_state(id, cols..., indexer_service_network_url) + + # Optimize for the indexer + ω = optimize_indexer(repo, indexer, parse(Float64, minimum_allocation_amount), parse(Int64, maximum_new_allocations)) + + # Create indexer rules + indexer_rules = create_rules!(id, indexer_service_network_url, ω, cols...) + + println.(indexer_rules) + + return nothing +end + +@main diff --git a/src/ActionQueue.jl b/src/ActionQueue.jl index 25e0890..d4e8cfc 100644 --- a/src/ActionQueue.jl +++ b/src/ActionQueue.jl @@ -1,4 +1,5 @@ module ActionQueue +include("action.jl") @enum ActionStatus begin queued @@ -52,7 +53,7 @@ function reallocate_actions( proposed_allocations::Dict{T,<:Real}, existing_allocations::Dict{T,T}, ) where {T<:AbstractString} - reallocate_ipfs = existing_ipfs ∩ proposed_ipfs + ipfses = reallocate_ipfs(existing_ipfs, proposed_ipfs) actions = map( ipfs -> structtodict( ReallocateActionInput( @@ -64,9 +65,9 @@ function reallocate_actions( "AllocationOpt", ), ), - reallocate_ipfs, + ipfses, ) - return actions, reallocate_ipfs + return actions, ipfses end function allocate_actions( @@ -74,7 +75,7 @@ function allocate_actions( reallocate_ipfs::Vector{T}, proposed_allocations::Dict{T,<:Real}, ) where {T<:AbstractString} - open_ipfs = setdiff(proposed_ipfs, reallocate_ipfs) + ipfses = open_ipfs(proposed_ipfs, reallocate_ipfs) actions = map( ipfs -> structtodict( AllocateActionInput( @@ -86,9 +87,9 @@ function allocate_actions( "AllocationOpt", ), ), - open_ipfs, + ipfses, ) - return actions, open_ipfs + return actions, ipfses end function unallocate_actions( @@ -97,7 +98,7 @@ function unallocate_actions( reallocate_ipfs::Vector{T}, frozenlist::Vector{T}, ) where {T<:AbstractString} - close_ipfs = setdiff(setdiff(existing_ipfs, reallocate_ipfs), frozenlist) + ipfses = close_ipfs(existing_ipfs, reallocate_ipfs, frozenlist) actions = map( ipfs -> structtodict( UnallocateActionInput( @@ -108,9 +109,9 @@ function unallocate_actions( "AllocationOpt", ), ), - close_ipfs, + ipfses, ) - return actions, close_ipfs + return actions, ipfses end end diff --git a/src/AllocationOpt.jl b/src/AllocationOpt.jl index 367cfbd..c5398d5 100644 --- a/src/AllocationOpt.jl +++ b/src/AllocationOpt.jl @@ -3,13 +3,14 @@ module AllocationOpt using CSV using GraphQLClient -export network_state, optimize_indexer, read_filterlists, push_allocations! +export network_state, optimize_indexer, read_filterlists, push_allocations!, create_rules! include("exceptions.jl") include("domainmodel.jl") include("query.jl") include("service.jl") include("ActionQueue.jl") +include("CLI.jl") """ function network_state(id, whitelist, blacklist, pinnedlist, frozenlist, indexer_service_network_url) @@ -137,8 +138,10 @@ function push_allocations!( existing_allocations = query_indexer_allocations( Client(indexer_service_network_url), indexer_id ) - existing_allocs = Dict(ipfshash.(existing_allocations) .=> id.(existing_allocations)) - existing_ipfs = ipfshash.(existing_allocations) + existing_allocs::Dict{String,String} = Dict( + ipfshash.(existing_allocations) .=> id.(existing_allocations) + ) + existing_ipfs::Vector{String} = ipfshash.(existing_allocations) proposed_ipfs = collect(keys(proposed_allocations)) # Generate ActionQueue inputs @@ -160,4 +163,39 @@ function push_allocations!( return response end +function create_rules!( + indexer_id::AbstractString, + indexer_service_network_url::T, + proposed_allocations::Dict{T,<:Real}, + whitelist::AbstractVector{T}, + blacklist::AbstractVector{T}, + pinnedlist::AbstractVector{T}, + frozenlist::AbstractVector{T}, +) where {T<:AbstractString} + actions = [] + + # Query existing allocations that are not frozen + existing_allocations = query_indexer_allocations( + Client(indexer_service_network_url), indexer_id + ) + existing_allocs = Dict(ipfshash.(existing_allocations) .=> id.(existing_allocations)) + existing_ipfs = ipfshash.(existing_allocations) + proposed_ipfs = collect(keys(proposed_allocations)) + + # Generate CLI commands + + reallocations, reallocate_ipfs = CLI.reallocate_actions( + proposed_ipfs, existing_ipfs, proposed_allocations, existing_allocs + ) + existing_ipfs ∩ proposed_ipfs + open_allocations, open_ipfs = CLI.allocate_actions( + proposed_ipfs, reallocate_ipfs, proposed_allocations + ) + close_allocations, close_ipfs = CLI.unallocate_actions( + existing_ipfs, reallocate_ipfs, frozenlist + ) + actions = vcat(reallocations, open_allocations, close_allocations) + return actions +end + end diff --git a/src/CLI.jl b/src/CLI.jl new file mode 100644 index 0000000..f7d0b30 --- /dev/null +++ b/src/CLI.jl @@ -0,0 +1,41 @@ +module CLI +using Formatting +include("action.jl") + +function reallocate_actions( + proposed_ipfs::Vector{T}, + existing_ipfs::Vector{T}, + proposed_allocations::Dict{T,<:Real}, + existing_allocations::Dict{T,T}, +) where {T<:AbstractString} + ipfses = reallocate_ipfs(existing_ipfs, proposed_ipfs) + actions = map( + ipfs -> + "\e[0mgraph indexer rules stop $(ipfs)\n\e[1m\e[38;2;255;0;0;249mCheck before submitting: \e[0mgraph indexer rules set $(ipfs) decisionBasis always allocationAmount $(format(proposed_allocations[ipfs]))", + ipfses, + ) + return actions, ipfses +end + +function allocate_actions( + proposed_ipfs::Vector{T}, + reallocate_ipfs::Vector{T}, + proposed_allocations::Dict{T,<:Real}, +) where {T<:AbstractString} + ipfses = open_ipfs(proposed_ipfs, reallocate_ipfs) + actions = map( + ipfs -> + "\e[0mgraph indexer rules set $(ipfs) decisionBasis always allocationAmount $(format(proposed_allocations[ipfs]))", + ipfses, + ) + return actions, ipfses +end + +function unallocate_actions( + existing_ipfs::Vector{T}, reallocate_ipfs::Vector{T}, frozenlist::Vector{T} +) where {T<:AbstractString} + ipfses = setdiff(setdiff(existing_ipfs, reallocate_ipfs), frozenlist) + actions = map(ipfs -> "\e[0mgraph indexer rules stop $(ipfs)", ipfses) + return actions, ipfses +end +end diff --git a/src/action.jl b/src/action.jl new file mode 100644 index 0000000..243bbeb --- /dev/null +++ b/src/action.jl @@ -0,0 +1,15 @@ +function reallocate_ipfs( + existing_ipfs::Vector{T}, proposed_ipfs::Vector{T} +) where {T<:AbstractString} + return existing_ipfs ∩ proposed_ipfs +end +function open_ipfs( + proposed_ipfs::Vector{T}, reallocate_ipfs::Vector{T} +) where {T<:AbstractString} + return setdiff(proposed_ipfs, reallocate_ipfs) +end +function close_ipfs( + existing_ipfs::Vector{T}, reallocate_ipfs::Vector{T}, frozenlist::Vector{T} +) where {T<:AbstractString} + return setdiff(setdiff(existing_ipfs, reallocate_ipfs), frozenlist) +end