feat: alternative script for generating indexing rules

Manifest.toml

@@ -121,6 +121,12 @@ version = "0.9.18"
 [[deps.FileWatching]]
 uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
 
+[[deps.Formatting]]
+deps = ["Printf"]
+git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8"
+uuid = "59287772-0a20-5a39-b81b-1366585eb4c0"
+version = "0.4.2"
+
 [[deps.Future]]
 deps = ["Random"]
 uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"

Project.toml

@@ -6,6 +6,7 @@ version = "1.9.0"
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 Debugger = "31a5f54b-26ea-5ae9-a837-f05ce5417438"
+Formatting = "59287772-0a20-5a39-b81b-1366585eb4c0"
 GraphQLClient = "09d831e3-9c21-47a9-bfd8-076871817219"
 JuliaFormatter = "98e50ef6-434e-11e9-1051-2b60c6c9e899"
 Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"

docs/src/index.md

@@ -20,7 +20,7 @@ pkg> add https://github.com/graphprotocol/AllocationOpt.jl/
 pkg> add Comonicon
 ```
 
-## Public Function
+## Public API
 
 We expose the following functions in our public API.
 

docs/src/usage.md

@@ -1,11 +1,37 @@
 # Usage
 
-## Configuring The Indexer Service
+## ActionQueue
 
 Set indexer-service flag `--serve-network-subgraph` to `true` so optimiser can pull data from the network subgraph at the indexer-service's network endpoint.
 If you would like to keep it false, you can instead supply an API url to `indexer_service_network_url`.
 
-## CLI Usage
+1. Download the [*allocationopt* script](https://raw.githubusercontent.com/graphprotocol/AllocationOpt.jl/main/scripts/allocationopt). For example, using `curl` or `wget`. Make sure you use the raw file!
+2. Make the *allocationopt* script executable.
+
+```bash
+$ chmod +x allocationopt
+```
+
+3. Run the *allocationopt* script with the *actionqueue* option. If this doesn't work, check that the shebang in the allocationopt file points to your julia executable.
+
+```bash
+$ ./scripts/allocationopt --help
+$ ./scripts/allocationopt actionqueue "0x6ac85b9d834b51b14a7b0ed849bb5199e04c05c5" test/example.csv 0.0 10 http://localhost:18000 http://localhost:7600/network
+```
+
+3. On Linux, you can further simplify the use of this script by symlinking it to your *.local/bin*
+
+```
+$ mkdir -p ~/.local/bin
+$ cd ~/.local/bin
+$ ln -s ~/projects/AllocationOpt.jl/scripts/allocationopt .  # Change to the path to the allocationopt script for you
+```
+
+You should now be able to run the *allocationopt* script from anywhere! Similarly to how you'd run *ls*.
+
+## Indexer Rules
+
+If you don't have the action queue set up yet, you can also run the optimiser by telling it to generate indexing rules.
 
 1. Download the [*allocationopt* script](https://raw.githubusercontent.com/graphprotocol/AllocationOpt.jl/main/scripts/allocationopt). For example, using `curl` or `wget`. Make sure you use the raw file!
 2. Make the *allocationopt* script executable.
@@ -14,11 +40,11 @@ If you would like to keep it false, you can instead supply an API url to `indexe
 $ chmod +x allocationopt
 ```
 
-3. Run the *allocationopt* script. If this doesn't work, check that the shebang in the allocationopt file points to your julia executable.
+3. Run the *allocationopt* script with the *rules* option. If this doesn't work, check that the shebang in the allocationopt file points to your julia executable.
 
 ```bash
 $ ./scripts/allocationopt --help
-$ ./scripts/allocationopt "0x6ac85b9d834b51b14a7b0ed849bb5199e04c05c5" test/example.csv 0.0 10 http://localhost:18000 http://localhost:7600/network
+$ ./scripts/allocationopt rules "0x6ac85b9d834b51b14a7b0ed849bb5199e04c05c5" test/example.csv 0.0 10 http://localhost:18000
 ```
 
 3. On Linux, you can further simplify the use of this script by symlinking it to your *.local/bin*

scripts/allocationopt

@@ -3,7 +3,7 @@ using Comonicon
 using AllocationOpt
 
 """
-Optimises an indexer's allocations.
+Optimises an indexer's allocations and pushes them to the action queue.
 
 # Arguments
 
@@ -15,7 +15,7 @@ Optimises an indexer's allocations.
 - `management_server_url`: The URL that exposes the indexer managment server, including the port. Must begin with http. Example: http://localhost:18000.
 - `indexer_service_network_url`: The URL that exposes the indexer service's network endpoint. Must begin with http. Example: http://localhost:7600/network.
 """
-@main function main(id, filepath, minimum_allocation_amount, maximum_new_allocations, management_server_url, indexer_service_network_url)
+@cast function actionqueue(id, filepath, minimum_allocation_amount, maximum_new_allocations, management_server_url, indexer_service_network_url)
     # Read subgraph lists defined in the file
     cols = read_filterlists(filepath)
 
@@ -26,9 +26,41 @@ Optimises an indexer's allocations.
     ω = optimize_indexer(repo, indexer, parse(Float64, minimum_allocation_amount), parse(Int64, maximum_new_allocations))
 
     # Push results to action queue
-    response = push_allocations!(id, management_server_url, indexer_service_network_url, ω, cols...)
+    _ = push_allocations!(id, management_server_url, indexer_service_network_url, ω, cols...)
 
     println("Done!")
 
-    return ω
+    return nothing
 end
+
+"""
+Optimises an indexer's allocations and generates indexer rules to change allocations.
+
+# Arguments
+
+- `id`: The id of the indexer to optimise.
+- `filepath`: A path to the CSV file that contains whitelist, blacklist, pinnedlist, frozenlist as columns.
+- `grtgas`: The maximum amount of GRT that you are willing to spend on each allocation transaction.
+- `minimum_allocation_amount`: The minimum amount of GRT that you are willing to allocate to a subgraph.
+- `allocation_lifetime`: The number of epoches you assume to open each allocations for. Lifetime 1 means an epoch should be closed in its second epoch
+- `indexer_service_network_url`: The URL that exposes the indexer service's network endpoint. Must begin with http. Example: http://localhost:7600/network.
+"""
+@cast function rules(id, filepath, minimum_allocation_amount, maximum_new_allocations, indexer_service_network_url)
+    # Read subgraph lists defined in the file
+    cols = read_filterlists(filepath)
+
+    # Pull network state from indexer service network endpoint
+    indexer, repo = network_state(id, cols..., indexer_service_network_url)
+
+    # Optimize for the indexer
+    ω = optimize_indexer(repo, indexer, parse(Float64, minimum_allocation_amount), parse(Int64, maximum_new_allocations))
+
+    # Create indexer rules
+    indexer_rules = create_rules!(id, indexer_service_network_url, ω, cols...)
+
+    println.(indexer_rules)
+
+    return nothing
+end
+
+@main

src/ActionQueue.jl

@@ -1,4 +1,5 @@
 module ActionQueue
+include("action.jl")
 
 @enum ActionStatus begin
     queued
@@ -52,7 +53,7 @@ function reallocate_actions(
     proposed_allocations::Dict{T,<:Real},
     existing_allocations::Dict{T,T},
 ) where {T<:AbstractString}
-    reallocate_ipfs = existing_ipfs ∩ proposed_ipfs
+    ipfses = reallocate_ipfs(existing_ipfs, proposed_ipfs)
     actions = map(
         ipfs -> structtodict(
             ReallocateActionInput(
@@ -64,17 +65,17 @@ function reallocate_actions(
                 "AllocationOpt",
             ),
         ),
-        reallocate_ipfs,
+        ipfses,
     )
-    return actions, reallocate_ipfs
+    return actions, ipfses
 end
 
 function allocate_actions(
     proposed_ipfs::Vector{T},
     reallocate_ipfs::Vector{T},
     proposed_allocations::Dict{T,<:Real},
 ) where {T<:AbstractString}
-    open_ipfs = setdiff(proposed_ipfs, reallocate_ipfs)
+    ipfses = open_ipfs(proposed_ipfs, reallocate_ipfs)
     actions = map(
         ipfs -> structtodict(
             AllocateActionInput(
@@ -86,9 +87,9 @@ function allocate_actions(
                 "AllocationOpt",
             ),
         ),
-        open_ipfs,
+        ipfses,
     )
-    return actions, open_ipfs
+    return actions, ipfses
 end
 
 function unallocate_actions(
@@ -97,7 +98,7 @@ function unallocate_actions(
     reallocate_ipfs::Vector{T},
     frozenlist::Vector{T},
 ) where {T<:AbstractString}
-    close_ipfs = setdiff(setdiff(existing_ipfs, reallocate_ipfs), frozenlist)
+    ipfses = close_ipfs(existing_ipfs, reallocate_ipfs, frozenlist)
     actions = map(
         ipfs -> structtodict(
             UnallocateActionInput(
@@ -108,9 +109,9 @@ function unallocate_actions(
                 "AllocationOpt",
             ),
         ),
-        close_ipfs,
+        ipfses,
     )
-    return actions, close_ipfs
+    return actions, ipfses
 end
 
 end

src/AllocationOpt.jl

@@ -3,13 +3,14 @@ module AllocationOpt
 using CSV
 using GraphQLClient
 
-export network_state, optimize_indexer, read_filterlists, push_allocations!
+export network_state, optimize_indexer, read_filterlists, push_allocations!, create_rules!
 
 include("exceptions.jl")
 include("domainmodel.jl")
 include("query.jl")
 include("service.jl")
 include("ActionQueue.jl")
+include("CLI.jl")
 
 """
     function network_state(id, whitelist, blacklist, pinnedlist, frozenlist, indexer_service_network_url)
@@ -137,8 +138,10 @@ function push_allocations!(
     existing_allocations = query_indexer_allocations(
         Client(indexer_service_network_url), indexer_id
     )
-    existing_allocs = Dict(ipfshash.(existing_allocations) .=> id.(existing_allocations))
-    existing_ipfs = ipfshash.(existing_allocations)
+    existing_allocs::Dict{String,String} = Dict(
+        ipfshash.(existing_allocations) .=> id.(existing_allocations)
+    )
+    existing_ipfs::Vector{String} = ipfshash.(existing_allocations)
     proposed_ipfs = collect(keys(proposed_allocations))
 
     # Generate ActionQueue inputs
@@ -160,4 +163,39 @@ function push_allocations!(
     return response
 end
 
+function create_rules!(
+    indexer_id::AbstractString,
+    indexer_service_network_url::T,
+    proposed_allocations::Dict{T,<:Real},
+    whitelist::AbstractVector{T},
+    blacklist::AbstractVector{T},
+    pinnedlist::AbstractVector{T},
+    frozenlist::AbstractVector{T},
+) where {T<:AbstractString}
+    actions = []
+
+    # Query existing allocations that are not frozen
+    existing_allocations = query_indexer_allocations(
+        Client(indexer_service_network_url), indexer_id
+    )
+    existing_allocs = Dict(ipfshash.(existing_allocations) .=> id.(existing_allocations))
+    existing_ipfs = ipfshash.(existing_allocations)
+    proposed_ipfs = collect(keys(proposed_allocations))
+
+    # Generate CLI commands
+
+    reallocations, reallocate_ipfs = CLI.reallocate_actions(
+        proposed_ipfs, existing_ipfs, proposed_allocations, existing_allocs
+    )
+    existing_ipfs ∩ proposed_ipfs
+    open_allocations, open_ipfs = CLI.allocate_actions(
+        proposed_ipfs, reallocate_ipfs, proposed_allocations
+    )
+    close_allocations, close_ipfs = CLI.unallocate_actions(
+        existing_ipfs, reallocate_ipfs, frozenlist
+    )
+    actions = vcat(reallocations, open_allocations, close_allocations)
+    return actions
+end
+
 end

src/CLI.jl

@@ -0,0 +1,41 @@
+module CLI
+using Formatting
+include("action.jl")
+
+function reallocate_actions(
+    proposed_ipfs::Vector{T},
+    existing_ipfs::Vector{T},
+    proposed_allocations::Dict{T,<:Real},
+    existing_allocations::Dict{T,T},
+) where {T<:AbstractString}
+    ipfses = reallocate_ipfs(existing_ipfs, proposed_ipfs)
+    actions = map(
+        ipfs ->
+            "\e[0mgraph indexer rules stop $(ipfs)\n\e[1m\e[38;2;255;0;0;249mCheck before submitting: \e[0mgraph indexer rules set $(ipfs) decisionBasis always allocationAmount $(format(proposed_allocations[ipfs]))",
+        ipfses,
+    )
+    return actions, ipfses
+end
+
+function allocate_actions(
+    proposed_ipfs::Vector{T},
+    reallocate_ipfs::Vector{T},
+    proposed_allocations::Dict{T,<:Real},
+) where {T<:AbstractString}
+    ipfses = open_ipfs(proposed_ipfs, reallocate_ipfs)
+    actions = map(
+        ipfs ->
+            "\e[0mgraph indexer rules set $(ipfs) decisionBasis always allocationAmount $(format(proposed_allocations[ipfs]))",
+        ipfses,
+    )
+    return actions, ipfses
+end
+
+function unallocate_actions(
+    existing_ipfs::Vector{T}, reallocate_ipfs::Vector{T}, frozenlist::Vector{T}
+) where {T<:AbstractString}
+    ipfses = setdiff(setdiff(existing_ipfs, reallocate_ipfs), frozenlist)
+    actions = map(ipfs -> "\e[0mgraph indexer rules stop $(ipfs)", ipfses)
+    return actions, ipfses
+end
+end

src/action.jl

@@ -0,0 +1,15 @@
+function reallocate_ipfs(
+    existing_ipfs::Vector{T}, proposed_ipfs::Vector{T}
+) where {T<:AbstractString}
+    return existing_ipfs ∩ proposed_ipfs
+end
+function open_ipfs(
+    proposed_ipfs::Vector{T}, reallocate_ipfs::Vector{T}
+) where {T<:AbstractString}
+    return setdiff(proposed_ipfs, reallocate_ipfs)
+end
+function close_ipfs(
+    existing_ipfs::Vector{T}, reallocate_ipfs::Vector{T}, frozenlist::Vector{T}
+) where {T<:AbstractString}
+    return setdiff(setdiff(existing_ipfs, reallocate_ipfs), frozenlist)
+end