Market scan

README.md

@@ -31,6 +31,7 @@
     - [Custom filters](#custom-filters)
     - [Custom ranking of proposals](#custom-ranking-of-proposals)
     - [Uploading local images to the provider](#uploading-local-images-to-the-provider)
+    - [Market scan](#market-scan)
   - [Going further](#going-further)
   - [More examples](#more-examples)
   - [Debugging](#debugging)
@@ -356,25 +357,26 @@ const order: MarketOrderSpec = {
 
 [Check the full example](./examples/advanced//local-image/)
 
-<!--
-TODO:
- ### Market scan
+### Market scan
 
 You can scan the market for available providers and their offers. This is useful when you want to see what's available
 before placing an order.
 
 ```ts
-await glm.market.scan(order).subscribe({
-  next: (proposal) => {
-    console.log("Received proposal from provider", proposal.provider.name);
-  },
-  complete: () => {
-    console.log("Market scan completed");
-  },
-});
+await glm.market
+  .scan(order)
+  .pipe(takeUntil(timer(10_000)))
+  .subscribe({
+    next: (scannedOffer) => {
+      console.log("Found offer from", scannedOffer.getProviderInfo().name);
+    },
+    complete: () => {
+      console.log("Market scan completed");
+    },
+  });
 ```
 
-[Check the full example](./examples/basic/market-scan.ts) -->
+[Check the full example](./examples/advanced/scan.ts)
 
 ## Going further
 

examples/advanced/scan.ts

@@ -0,0 +1,64 @@
+/**
+ * This example demonstrates how to scan the market for providers that meet specific requirements.
+ */
+import { GolemNetwork, ScanOptions } from "@golem-sdk/golem-js";
+import { last, map, scan, takeUntil, tap, timer } from "rxjs";
+
+// What providers are we looking for?
+const scanOptions: ScanOptions = {
+  subnetTag: "public",
+  workload: {
+    engine: "vm",
+    minCpuCores: 4,
+    minMemGib: 8,
+    minCpuThreads: 8,
+    capabilities: ["vpn"],
+    minStorageGib: 16,
+  },
+};
+
+(async () => {
+  const glm = new GolemNetwork();
+  await glm.connect();
+  const spec = glm.market.buildScanSpecification(scanOptions);
+
+  /* For advanced users: you can also add properties and constraints manually:
+  spec.properties.push({
+    key: "golem.inf.cpu.architecture",
+    value: "x86_64",
+  });
+  */
+
+  const SCAN_DURATION_MS = 10_000;
+
+  console.log(`Scanning for ${SCAN_DURATION_MS / 1000} seconds...`);
+  glm.market
+    .scan(spec)
+    .pipe(
+      tap((scannedOffer) => {
+        console.log("Found offer from", scannedOffer.getProviderInfo().name);
+      }),
+      // calculate the cost of an hour of work
+      map(
+        (scannedOffer) =>
+          scannedOffer.pricing.start + //
+          scannedOffer.pricing.cpuSec * 3600 +
+          scannedOffer.pricing.envSec * 3600,
+      ),
+      // calculate the running average
+      scan((total, cost) => total + cost, 0),
+      map((totalCost, index) => totalCost / (index + 1)),
+      // stop scanning after SCAN_DURATION_MS
+      takeUntil(timer(SCAN_DURATION_MS)),
+      last(),
+    )
+    .subscribe({
+      next: (averageCost) => {
+        console.log("Average cost for an hour of work:", averageCost.toFixed(6), "GLM");
+      },
+      complete: () => {
+        console.log("Scan completed, shutting down...");
+        glm.disconnect();
+      },
+    });
+})();

examples/package.json

@@ -16,6 +16,7 @@
     "advanced-payment-filters": "tsx advanced/payment-filters.ts",
     "advanced-proposal-filters": "tsx advanced/proposal-filter.ts",
     "advanced-proposal-predefined-filter": "tsx advanced/proposal-predefined-filter.ts",
+    "advanced-scan": "tsx advanced/scan.ts",
     "local-image": "tsx advanced/local-image/serveLocalGvmi.ts",
     "deployment": "tsx experimental/deployment/new-api.ts",
     "market-scan": "tsx market/scan.ts",

src/market/api.ts

@@ -18,6 +18,7 @@ import {
   AgreementTerminatedEvent,
 } from "./agreement";
 import { AgreementOptions } from "./agreement/agreement";
+import { ScanSpecification, ScannedOffer } from "./scan";
 
 export type MarketEvents = {
   demandSubscriptionStarted: (demand: Demand) => void;
@@ -140,4 +141,9 @@ export interface IMarketApi {
    * Retrieves the state of an agreement based on the provided agreement ID.
    */
   getAgreementState(id: string): Promise<AgreementState>;
+
+  /**
+   * Scan the market for offers that match the given specification.
+   */
+  scan(scanSpecification: ScanSpecification): Observable<ScannedOffer>;
 }

src/market/index.ts

@@ -12,3 +12,4 @@ export { BasicDemandDirector } from "./demand/directors/basic-demand-director";
 export { PaymentDemandDirector } from "./demand/directors/payment-demand-director";
 export { WorkloadDemandDirector } from "./demand/directors/workload-demand-director";
 export * from "./proposal/market-proposal-event";
+export * from "./scan";

src/market/market.module.ts

@@ -42,6 +42,7 @@ import { MarketOrderSpec } from "../golem-network";
 import { INetworkApi, NetworkModule } from "../network";
 import { AgreementOptions } from "./agreement/agreement";
 import { Concurrency } from "../lease-process";
+import { ScanDirector, ScanOptions, ScanSpecification, ScannedOffer } from "./scan";
 
 export type DemandEngine = "vm" | "vm-nvidia" | "wasmtime";
 
@@ -82,6 +83,13 @@ export interface MarketModule {
    */
   buildDemandDetails(options: BuildDemandOptions, allocation: Allocation): Promise<DemandSpecification>;
 
+  /**
+   * Build a ScanSpecification that can be used to scan the market for offers.
+   * The difference between this method and `buildDemandDetails` is that this method does not require an
+   * allocation, doesn't set payment related properties and doesn't provide any defaults.
+   */
+  buildScanSpecification(options: ScanOptions): ScanSpecification;
+
   /**
    * Publishes the demand to the market and handles refreshing it when needed.
    * Each time the demand is refreshed, a new demand is emitted by the observable.
@@ -189,6 +197,11 @@ export interface MarketModule {
    * Fetch the most up-to-date agreement details from the yagna
    */
   fetchAgreement(agreementId: string): Promise<Agreement>;
+
+  /**
+   * Scan the market for offers that match the given demand specification.
+   */
+  scan(scanSpecification: ScanSpecification): Observable<ScannedOffer>;
 }
 
 /**
@@ -265,6 +278,13 @@ export class MarketModuleImpl implements MarketModule {
     return new DemandSpecification(builder.getProduct(), allocation.paymentPlatform, basicConfig.expirationSec);
   }
 
+  buildScanSpecification(options: ScanOptions): ScanSpecification {
+    const builder = new DemandBodyBuilder();
+    const director = new ScanDirector(options);
+    director.apply(builder);
+    return builder.getProduct();
+  }
+
   /**
    * Augments the user-provided options with additional logic
    *
@@ -649,4 +669,8 @@ export class MarketModuleImpl implements MarketModule {
     }
     return isPriceValid;
   }
+
+  scan(scanSpecification: ScanSpecification): Observable<ScannedOffer> {
+    return this.deps.marketApi.scan(scanSpecification);
+  }
 }

src/market/scan/index.ts

@@ -0,0 +1,3 @@
+export * from "./types";
+export * from "./scan-director";
+export * from "./scanned-proposal";

src/market/scan/scan-director.ts

@@ -0,0 +1,81 @@
+import { ComparisonOperator, DemandBodyBuilder } from "../demand";
+import { ScanOptions } from "./types";
+
+export class ScanDirector {
+  constructor(private options: ScanOptions) {}
+
+  public async apply(builder: DemandBodyBuilder) {
+    this.addWorkloadDecorations(builder);
+    this.addGenericDecorations(builder);
+    this.addManifestDecorations(builder);
+    this.addPaymentDecorations(builder);
+  }
+
+  private addPaymentDecorations(builder: DemandBodyBuilder): void {
+    if (this.options.payment?.debitNotesAcceptanceTimeoutSec) {
+      builder.addProperty(
+        "golem.com.payment.debit-notes.accept-timeout?",
+        this.options.payment?.debitNotesAcceptanceTimeoutSec,
+      );
+    }
+    if (this.options.payment?.midAgreementDebitNoteIntervalSec) {
+      builder.addProperty(
+        "golem.com.scheme.payu.debit-note.interval-sec?",
+        this.options.payment?.midAgreementDebitNoteIntervalSec,
+      );
+    }
+    if (this.options.payment?.midAgreementPaymentTimeoutSec) {
+      builder.addProperty(
+        "golem.com.scheme.payu.payment-timeout-sec?",
+        this.options.payment?.midAgreementPaymentTimeoutSec,
+      );
+    }
+  }
+
+  private addWorkloadDecorations(builder: DemandBodyBuilder): void {
+    if (this.options.workload?.engine) {
+      builder.addConstraint("golem.runtime.name", this.options.workload?.engine);
+    }
+    if (this.options.workload?.capabilities)
+      this.options.workload?.capabilities.forEach((cap) => builder.addConstraint("golem.runtime.capabilities", cap));
+
+    if (this.options.workload?.minMemGib) {
+      builder.addConstraint("golem.inf.mem.gib", this.options.workload?.minMemGib, ComparisonOperator.GtEq);
+    }
+    if (this.options.workload?.minStorageGib) {
+      builder.addConstraint("golem.inf.storage.gib", this.options.workload?.minStorageGib, ComparisonOperator.GtEq);
+    }
+    if (this.options.workload?.minCpuThreads) {
+      builder.addConstraint("golem.inf.cpu.threads", this.options.workload?.minCpuThreads, ComparisonOperator.GtEq);
+    }
+    if (this.options.workload?.minCpuCores) {
+      builder.addConstraint("golem.inf.cpu.cores", this.options.workload?.minCpuCores, ComparisonOperator.GtEq);
+    }
+  }
+
+  private addGenericDecorations(builder: DemandBodyBuilder): void {
+    if (this.options.subnetTag) {
+      builder
+        .addProperty("golem.node.debug.subnet", this.options.subnetTag)
+        .addConstraint("golem.node.debug.subnet", this.options.subnetTag);
+    }
+
+    if (this.options.expirationSec) {
+      builder.addProperty("golem.srv.comp.expiration", Date.now() + this.options.expirationSec * 1000);
+    }
+  }
+
+  private addManifestDecorations(builder: DemandBodyBuilder): void {
+    if (!this.options.workload?.manifest) return;
+    builder.addProperty("golem.srv.comp.payload", this.options.workload?.manifest);
+    if (this.options.workload?.manifestSig) {
+      builder.addProperty("golem.srv.comp.payload.sig", this.options.workload?.manifestSig);
+    }
+    if (this.options.workload?.manifestSigAlgorithm) {
+      builder.addProperty("golem.srv.comp.payload.sig.algorithm", this.options.workload?.manifestSigAlgorithm);
+    }
+    if (this.options.workload?.manifestCert) {
+      builder.addProperty("golem.srv.comp.payload.cert", this.options.workload?.manifestCert);
+    }
+  }
+}

src/market/scan/scanned-proposal.ts

@@ -0,0 +1,91 @@
+import { PricingInfo, ProposalProperties } from "../proposal";
+import { GolemInternalError } from "../../shared/error/golem-error";
+
+// Raw response from yagna
+// TODO: add to ya-client
+type ScannedOfferDTO = {
+  properties: ProposalProperties;
+  constraints: string;
+  offerId: string;
+  providerId: string;
+  timestamp: string;
+};
+
+export class ScannedOffer {
+  constructor(private readonly model: ScannedOfferDTO) {}
+
+  get properties(): ProposalProperties {
+    return this.model.properties;
+  }
+
+  get constraints(): string {
+    return this.model.constraints;
+  }
+
+  get pricing(): PricingInfo {
+    const usageVector = this.properties["golem.com.usage.vector"];
+    const priceVector = this.properties["golem.com.pricing.model.linear.coeffs"];
+
+    if (!usageVector) {
+      throw new GolemInternalError(
+        "The proposal does not contain 'golem.com.usage.vector' property. We can't estimate the costs.",
+      );
+    }
+
+    if (!priceVector) {
+      throw new GolemInternalError(
+        "The proposal does not contain 'golem.com.pricing.model.linear.coeffs' property. We can't estimate costs.",
+      );
+    }
+
+    const envIdx = usageVector.findIndex((ele) => ele === "golem.usage.duration_sec");
+    const cpuIdx = usageVector.findIndex((ele) => ele === "golem.usage.cpu_sec");
+
+    const envSec = priceVector[envIdx] ?? 0.0;
+    const cpuSec = priceVector[cpuIdx] ?? 0.0;
+    const start = priceVector[priceVector.length - 1];
+
+    return {
+      cpuSec,
+      envSec,
+      start,
+    };
+  }
+
+  getProviderInfo() {
+    return {
+      id: this.model.providerId,
+      name: this.properties["golem.node.id.name"] || "<unknown>",
+    };
+  }
+  get transferProtocol() {
+    return this.properties["golem.activity.caps.transfer.protocol"];
+  }
+  get cpuBrand() {
+    return this.properties["golem.inf.cpu.brand"];
+  }
+  get cpuCapabilities() {
+    return this.properties["golem.inf.cpu.capabilities"];
+  }
+  get cpuCores() {
+    return this.properties["golem.inf.cpu.cores"];
+  }
+  get cpuThreads() {
+    return this.properties["golem.inf.cpu.threads"];
+  }
+  get memory() {
+    return this.properties["golem.inf.mem.gib"];
+  }
+  get storage() {
+    return this.properties["golem.inf.storage.gib"];
+  }
+  get publicNet() {
+    return this.properties["golem.node.net.is-public"];
+  }
+  get runtimeCapabilities() {
+    return this.properties["golem.runtime.capabilities"];
+  }
+  get runtimeName() {
+    return this.properties["golem.runtime.name"];
+  }
+}

src/market/scan/types.ts

@@ -0,0 +1,17 @@
+import { BuildDemandOptions, DemandProperty } from "../demand";
+
+// recursively make all properties optional (but not array members)
+type DeepPartial<T> = T extends object
+  ? T extends Array<unknown>
+    ? T
+    : {
+        [P in keyof T]?: DeepPartial<T[P]>;
+      }
+  : T;
+
+export type ScanOptions = DeepPartial<BuildDemandOptions>;
+
+export type ScanSpecification = {
+  properties: DemandProperty[];
+  constraints: string[];
+};