From 413f5b895652215bc5518d8923133e0856d35d7a Mon Sep 17 00:00:00 2001 From: Lucas Pollice Date: Sat, 17 Feb 2024 14:48:34 -0500 Subject: [PATCH 1/3] fix bug with new artifact not being scanned --- package.json | 2 +- packages/automation/src/util/scraper.ts | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 29f5ec2..4790848 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "genshin-locker", "private": true, "productName": "genshin-locker", - "version": "1.2.1", + "version": "1.2.2", "description": "Genshin artifact scraper and manager", "main": "packages/main/dist/index.cjs", "scripts": { diff --git a/packages/automation/src/util/scraper.ts b/packages/automation/src/util/scraper.ts index d9b62c5..0548c0b 100644 --- a/packages/automation/src/util/scraper.ts +++ b/packages/automation/src/util/scraper.ts @@ -89,12 +89,13 @@ export const getSubstats = (txts: string[]): SubStat[] => { export const getArtifactSet = (txt: string): SetKey => { const normalizedTxt = txt.toLowerCase().replaceAll(/[^a-z]+/g, '') const artifactData = datamine.artifacts - if (!(normalizedTxt in artifactData)) { + const artifactEntry = Object.entries(artifactData) + // Some artifact set names span multiple lines and get cut off by the scan + .find(([key]) => key.startsWith(normalizedTxt)) + if (!artifactEntry) { throw Error(`"${normalizedTxt}" not a valid artifact set`) } - return artifactData[normalizedTxt as keyof typeof artifactData][ - 'GOOD' - ] as SetKey + return stringToEnum(artifactEntry[1]['GOOD'], SetKey) } export const getStatKey = (txt: string): string => { let normalizedTxt = txt.toLowerCase().replaceAll(/[^a-z]+/g, '') From 1be8f9f71af6cf2a1d77c011d2667fd760ea9ddc Mon Sep 17 00:00:00 2001 From: Lucas Pollice Date: Sat, 17 Feb 2024 20:42:52 -0500 Subject: [PATCH 2/3] refactor scripts, remove dead code --- package.json | 2 +- .../database/collections/default/index.ts | 1 - .../database/collections/default/v0.ts | 51 ---- .../database/collections/default/v1.ts | 75 ------ .../database/collections/default/v2.ts | 97 -------- .../src/scoring/database/collections/index.ts | 11 - .../src/scoring/database/collections/types.ts | 8 - .../src/scoring/database/database.ts | 35 --- .../automation/src/scoring/database/index.ts | 1 - packages/automation/src/scoring/index.ts | 1 - packages/scripts/bin/index.mjs | 11 + packages/scripts/bin/types.mjs | 26 +++ .../scripts/bin/update-electron-vendors.mjs | 34 +++ packages/scripts/package.json | 16 +- packages/scripts/src/index.ts | 14 -- packages/scripts/src/popularity.ts | 219 ------------------ packages/scripts/src/rarity.ts | 64 ----- packages/scripts/src/types.ts | 26 --- .../scripts/src/update-electron-vendors.mjs | 18 -- packages/scripts/tsconfig.json | 2 +- yarn.lock | 28 +++ 21 files changed, 106 insertions(+), 634 deletions(-) delete mode 100644 packages/automation/src/scoring/database/collections/default/index.ts delete mode 100644 packages/automation/src/scoring/database/collections/default/v0.ts delete mode 100644 packages/automation/src/scoring/database/collections/default/v1.ts delete mode 100644 packages/automation/src/scoring/database/collections/default/v2.ts delete mode 100644 packages/automation/src/scoring/database/collections/index.ts delete mode 100644 packages/automation/src/scoring/database/collections/types.ts delete mode 100644 packages/automation/src/scoring/database/database.ts delete mode 100644 packages/automation/src/scoring/database/index.ts create mode 100644 packages/scripts/bin/index.mjs create mode 100644 packages/scripts/bin/types.mjs create mode 100644 packages/scripts/bin/update-electron-vendors.mjs delete mode 100644 packages/scripts/src/index.ts delete mode 100644 packages/scripts/src/popularity.ts delete mode 100644 packages/scripts/src/rarity.ts delete mode 100644 packages/scripts/src/types.ts delete mode 100644 packages/scripts/src/update-electron-vendors.mjs diff --git a/package.json b/package.json index 4790848..3ec991e 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "compile:unpacked": "yarn build && electron-builder build --config .electron-builder.config.js --dir --config.asar=false", "watch": "node watch.mjs", "prepare": "husky install", - "postinstall": "lerna run set-env-targets --scope=@gl/scripts" + "postinstall": "cross-env ELECTRON_RUN_AS_NODE=1 electron packages/scripts/bin/index.mjs update-electron-vendors" }, "keywords": [], "author": { diff --git a/packages/automation/src/scoring/database/collections/default/index.ts b/packages/automation/src/scoring/database/collections/default/index.ts deleted file mode 100644 index 523b9ef..0000000 --- a/packages/automation/src/scoring/database/collections/default/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './v2' diff --git a/packages/automation/src/scoring/database/collections/default/v0.ts b/packages/automation/src/scoring/database/collections/default/v0.ts deleted file mode 100644 index 7bf38c0..0000000 --- a/packages/automation/src/scoring/database/collections/default/v0.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { - RxJsonSchema, - toTypedRxJsonSchema, - ExtractDocumentTypeFromTypedRxJsonSchema, - MigrationStrategies, -} from 'rxdb' - -const schemaLiteral = { - title: 'precalculation schema', - description: 'precalculated scores', - version: 0, - primaryKey: { - key: 'id', - fields: ['set', 'slot', 'main', 'sub'], - separator: '|', - }, - type: 'object', - properties: { - id: { - type: 'string', - maxLength: 100, - }, - set: { - type: 'string', - }, - slot: { - type: 'string', - }, - main: { - type: 'string', - }, - sub: { - type: 'string', - }, - popularity: { - type: 'number', - default: 0, - }, - }, - required: ['set', 'slot', 'main', 'sub'], -} as const - -const schemaTyped = toTypedRxJsonSchema(schemaLiteral) - -export type RxDocType = ExtractDocumentTypeFromTypedRxJsonSchema< - typeof schemaTyped -> - -export const schema: RxJsonSchema = schemaTyped - -export const migrationStrategies: MigrationStrategies = {} diff --git a/packages/automation/src/scoring/database/collections/default/v1.ts b/packages/automation/src/scoring/database/collections/default/v1.ts deleted file mode 100644 index 3ac06fd..0000000 --- a/packages/automation/src/scoring/database/collections/default/v1.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { - RxJsonSchema, - toTypedRxJsonSchema, - ExtractDocumentTypeFromTypedRxJsonSchema, - MigrationStrategies, -} from 'rxdb' - -import { MigrationStrategy } from '../types' - -import { - RxDocType as LastRxDocType, - migrationStrategies as previousMigrationStrategies, -} from './v0' - -const VERSION = 1 - -const schemaLiteral = { - title: 'precalculation schema', - description: 'precalculated scores', - version: VERSION, - primaryKey: { - key: 'id', - fields: ['set', 'slot', 'main', 'sub'], - separator: '|', - }, - type: 'object', - properties: { - id: { - type: 'string', - maxLength: 100, - }, - set: { - type: 'string', - }, - slot: { - type: 'string', - }, - main: { - type: 'string', - }, - sub: { - type: 'string', - }, - popularity: { - type: 'number', - }, - rarity: { - type: 'number', - }, - }, - required: ['set', 'slot', 'main', 'sub', 'popularity', 'rarity'], -} as const - -const schemaTyped = toTypedRxJsonSchema(schemaLiteral) - -export type RxDocType = ExtractDocumentTypeFromTypedRxJsonSchema< - typeof schemaTyped -> - -export const schema: RxJsonSchema = schemaTyped - -const migrationStrategy: MigrationStrategy = ( - oldDoc -) => { - return { - ...oldDoc, - popularity: oldDoc.popularity ?? 0, - rarity: 0, - } -} - -export const migrationStrategies: MigrationStrategies = { - ...previousMigrationStrategies, - [VERSION]: migrationStrategy, -} diff --git a/packages/automation/src/scoring/database/collections/default/v2.ts b/packages/automation/src/scoring/database/collections/default/v2.ts deleted file mode 100644 index 2dabc75..0000000 --- a/packages/automation/src/scoring/database/collections/default/v2.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { - RxJsonSchema, - toTypedRxJsonSchema, - ExtractDocumentTypeFromTypedRxJsonSchema, - MigrationStrategies, - RxCollectionCreator, -} from 'rxdb' - -import { MigrationStrategy } from '../types' - -import { - RxDocType as LastRxDocType, - migrationStrategies as previousMigrationStrategies, -} from './v1' - -const VERSION = 2 - -const scoreProperties = { - rarity: { - type: 'number', - minimum: 0, - maximum: 1, - }, - popularity: { - type: 'number', - minimum: 0, - multipleOf: 1, - }, -} as const - -export type CachedScores = keyof typeof scoreProperties -export const cachedScoreTypes = Object.keys(scoreProperties) as CachedScores[] - -const schemaLiteral = { - title: 'precalculation schema', - description: 'precalculated scores', - version: VERSION, - primaryKey: { - key: 'id', - fields: ['set', 'slot', 'main', 'sub'], - separator: '|', - }, - type: 'object', - properties: { - id: { - type: 'string', - maxLength: 100, - }, - set: { - type: 'string', - }, - slot: { - type: 'string', - }, - main: { - type: 'string', - }, - sub: { - type: 'string', - }, - ...scoreProperties, - }, - required: [ - 'set', - 'slot', - 'main', - 'sub', - ...(Object.keys(scoreProperties) as CachedScores[]), - ], -} as const // satisfies Parameters[0] - -const schemaTyped = toTypedRxJsonSchema(schemaLiteral) - -export type RxDocType = ExtractDocumentTypeFromTypedRxJsonSchema< - typeof schemaTyped -> - -export const schema: RxJsonSchema = schemaTyped - -const migrationStrategy: MigrationStrategy = ( - oldDoc -) => { - return { - ...oldDoc, - } -} - -export const migrationStrategies: MigrationStrategies = { - ...previousMigrationStrategies, - [VERSION]: migrationStrategy, -} - -export const collection: RxCollectionCreator = { - schema, - migrationStrategies, - localDocuments: true, -} diff --git a/packages/automation/src/scoring/database/collections/index.ts b/packages/automation/src/scoring/database/collections/index.ts deleted file mode 100644 index f5dd34e..0000000 --- a/packages/automation/src/scoring/database/collections/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { RxCollection, RxCollectionCreator } from 'rxdb' - -import * as Default from './default' - -export type Collections = { - default: RxCollection -} - -export const collections: Record = { - default: Default.collection, -} diff --git a/packages/automation/src/scoring/database/collections/types.ts b/packages/automation/src/scoring/database/collections/types.ts deleted file mode 100644 index 6a8eaff..0000000 --- a/packages/automation/src/scoring/database/collections/types.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { OldRxCollection } from 'rxdb' - -type MaybePromise = T | Promise - -export type MigrationStrategy = ( - oldDocumentData: OldDocData, - oldRxCollection: OldRxCollection -) => MaybePromise diff --git a/packages/automation/src/scoring/database/database.ts b/packages/automation/src/scoring/database/database.ts deleted file mode 100644 index 5dace71..0000000 --- a/packages/automation/src/scoring/database/database.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { LokiFsAdapter, LokiMemoryAdapter } from 'lokijs' -import { createRxDatabase, addRxPlugin, RxDatabase } from 'rxdb' -import { RxDBDevModePlugin } from 'rxdb/plugins/dev-mode' -import { RxDBJsonDumpPlugin } from 'rxdb/plugins/json-dump' -import { RxDBLocalDocumentsPlugin } from 'rxdb/plugins/local-documents' -import { getRxStorageLoki } from 'rxdb/plugins/lokijs' -import { RxDBMigrationPlugin } from 'rxdb/plugins/migration' -import { RxDBQueryBuilderPlugin } from 'rxdb/plugins/query-builder' -import { RxDBUpdatePlugin } from 'rxdb/plugins/update' - -import { Collections, collections } from './collections' - -addRxPlugin(RxDBDevModePlugin) -addRxPlugin(RxDBJsonDumpPlugin) -addRxPlugin(RxDBLocalDocumentsPlugin) -addRxPlugin(RxDBMigrationPlugin) -addRxPlugin(RxDBQueryBuilderPlugin) -addRxPlugin(RxDBUpdatePlugin) - -let db: Promise> -export async function getDatabase(importData = true) { - if (db) { - return db - } - db = createRxDatabase({ - name: 'default', - storage: getRxStorageLoki({ - adapter: importData ? new LokiFsAdapter() : new LokiMemoryAdapter(), - }), - }).then(async (newDB) => { - await newDB.addCollections(collections) - return newDB - }) - return db -} diff --git a/packages/automation/src/scoring/database/index.ts b/packages/automation/src/scoring/database/index.ts deleted file mode 100644 index e1ddd7b..0000000 --- a/packages/automation/src/scoring/database/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './database' diff --git a/packages/automation/src/scoring/index.ts b/packages/automation/src/scoring/index.ts index 12226cd..abf400f 100644 --- a/packages/automation/src/scoring/index.ts +++ b/packages/automation/src/scoring/index.ts @@ -1,4 +1,3 @@ export * from './logic' export * from './types' -export { getDatabase } from './database' export * from './const' diff --git a/packages/scripts/bin/index.mjs b/packages/scripts/bin/index.mjs new file mode 100644 index 0000000..ff98c61 --- /dev/null +++ b/packages/scripts/bin/index.mjs @@ -0,0 +1,11 @@ +#!/usr/bin/env node +import { hideBin } from 'yargs/helpers' +import yargs from 'yargs/yargs' + +import * as updateElectronVendors from './update-electron-vendors.mjs' + +yargs(hideBin(process.argv)) + .command(updateElectronVendors.command) + .demandCommand() + .help() + .parse() diff --git a/packages/scripts/bin/types.mjs b/packages/scripts/bin/types.mjs new file mode 100644 index 0000000..391f40e --- /dev/null +++ b/packages/scripts/bin/types.mjs @@ -0,0 +1,26 @@ +/** + * @typedef {import("yargs").Options} Options + * @typedef {import("yargs").ArgumentsCamelCase} ArgumentsCamelCase + * @typedef {{ [key: string]: Options; }} OptionsDict + */ + +/** + * @template {OptionsDict} O + * @typedef {import("yargs").CommandModule>} CommandModule + */ + +/** + * @template {OptionsDict} O + * @typedef {CommandModule & { + * builder?: O & OptionsDict; + * handler: (args: import("yargs").ArgumentsCamelCase>) => void | Promise; + * }} MyCommandModule + */ + +/** + * Due to a limitation with typescript, we need this function to infer the generic portion of MyCommandModule. + * @template {OptionsDict} T + * @param {MyCommandModule} module + * @returns {import("yargs").CommandModule>} + */ +export const asCommand = (module) => module diff --git a/packages/scripts/bin/update-electron-vendors.mjs b/packages/scripts/bin/update-electron-vendors.mjs new file mode 100644 index 0000000..0499895 --- /dev/null +++ b/packages/scripts/bin/update-electron-vendors.mjs @@ -0,0 +1,34 @@ +import { writeFileSync } from 'fs' +import path from 'path' + +import { workspaceRoot } from '@nx/devkit' + +import { asCommand } from './types.mjs' + +const electronRelease = process.versions + +const node = electronRelease.node.split('.')[0] +const chrome = electronRelease.v8.split('.').splice(0, 2).join('') + +export const command = asCommand({ + command: 'update-electron-vendors', + describe: + 'Generate the node and chrome targets for the current version of electron.', + builder: {}, + handler: () => { + if (!process.versions['electron']) { + throw new Error( + `This script should be run in electron context. Example: \`ELECTRON_RUN_AS_NODE=1 electron update-electron-vendors.mjs\`` + ) + } + writeFileSync( + path.join(workspaceRoot, '.electron-vendors.cache.json'), + JSON.stringify({ chrome, node }) + ) + writeFileSync( + path.join(workspaceRoot, '.browserslistrc'), + `Chrome ${chrome}`, + 'utf8' + ) + }, +}) diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 6fc81a4..6187346 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -5,13 +5,7 @@ "author": "Lucas Pollice ", "homepage": "https://github.com/Jugbot/genshin-locker#readme", "license": "ISC", - "main": "src/index.ts", - "directories": { - "src": "src" - }, - "files": [ - "src" - ], + "bin": "bin/index.mjs", "repository": { "type": "git", "url": "git+https://github.com/Jugbot/genshin-locker.git" @@ -22,13 +16,13 @@ "lint:write": "eslint . --max-warnings=0 --ignore-path=\"../../.gitignore\" --fix", "lint": "eslint . --max-warnings=0 --ignore-path=\"../../.gitignore\"", "check-types": "tsc --noEmit", - "script": "vite-node --script src", - "set-env-targets": "cross-env ELECTRON_RUN_AS_NODE=1 electron src/update-electron-vendors.mjs" + "scripts": "scripts" }, "bugs": { "url": "https://github.com/Jugbot/genshin-locker/issues" }, "dependencies": { + "@nx/devkit": "^18.0.4", "@gl/automation": "*", "@gl/types": "*", "cli-progress": "^3.11.2", @@ -42,7 +36,7 @@ "electron": "^24.0.0", "ts-node": "^10.9.1", "typescript": "^4.9.0", - "vite-node": "^0.33.0", - "vite": "4.2.1" + "vite": "4.2.1", + "vite-node": "^0.33.0" } } diff --git a/packages/scripts/src/index.ts b/packages/scripts/src/index.ts deleted file mode 100644 index 877aecd..0000000 --- a/packages/scripts/src/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { hideBin } from 'yargs/helpers' -import yargs from 'yargs/yargs' - -import * as popularity from './popularity' -import * as rarity from './rarity' -// import * as vendors from './update-electron-vendors' - -yargs(hideBin(process.argv)) - .command(rarity.command) - .command(popularity.command) - // .command(vendors.command) - .demandCommand() - .help() - .parse() diff --git a/packages/scripts/src/popularity.ts b/packages/scripts/src/popularity.ts deleted file mode 100644 index ad5c33d..0000000 --- a/packages/scripts/src/popularity.ts +++ /dev/null @@ -1,219 +0,0 @@ -import { getDatabase, getArtifactSet, getStatKey } from '@gl/automation' -import { SingleBar } from 'cli-progress' -import fetch from 'node-fetch' - -import { asCommand } from './types' - -const timeInSecondsUTC = (date: Date) => { - return Math.trunc(date.getTime() / 1000) -} -const currentTimeSeconds = () => { - return timeInSecondsUTC(new Date()) -} -// TODO: scrape data from other languages -type Payload = { - data: { - next_page_token: string - list: Array<{ - id: string - account_uid: string - nickname: string - avatar_url: string - level: number - title: string - tag_ids: Array - avatar_group: Array<{ - group: Array<{ - id: number - name: string - icon: string - element: number - level: number - weapon_cat_id: number - weapon: { - id: number - name: string - icon: string - level: number - cat_id: number - wiki_url: string - } - set_list: Array<{ - id: number - name: string - icon: string - level: number - attr_id: number - cat_id: number - wiki_url: string - }> - wiki_url: string - avatar_tag: { - id: number - name: string - color: string - } - first_attr: Array<{ - cat_id: number - id: number - name: string - }> - secondary_attr: [] - secondary_attr_name: Array<{ - id: number - name: string - }> - head_icon: string - pc_icon: string - }> - }> - quick_avatar: [] - description: string - like_cnt: number - comment_cnt: number - favour_cnt: number - view_cnt: number - is_favour: boolean - is_like: boolean - status: number - is_deleted: boolean - created_at: string - group_cnt: number - trans: boolean - origin_title: string - origin_desc: string - tag_name: [] - trans_from: string - }> - } -} - -async function* fetchLineupSimulatorBuilds(limit = 1000) { - let nextPageToken = '' - while (nextPageToken !== null) { - const request = new URL( - 'https://sg-public-api.hoyolab.com/event/simulatoros/lineup/index' - ) - request.searchParams.set('limit', `${limit}`) - request.searchParams.set('next_page_token', nextPageToken) - request.searchParams.set('tag_id', '2') - request.searchParams.set('roles', '') - request.searchParams.set('order', 'CreatedTime') - request.searchParams.set('lang', 'en-us') - try { - const data = await fetch(request) - const jsonData = (await data.json()) as Payload - nextPageToken = jsonData.data['next_page_token'] - yield* jsonData.data['list'] - } catch (e) { - console.error(e) - } - } -} - -const LOCAL_DOC_KEY = 'sync-time' -type LocalDoc = { - time: number -} - -async function createStatistics() { - const db = await getDatabase() - let localDoc = await db.default.getLocal(LOCAL_DOC_KEY) - if (localDoc === null) { - localDoc = await db.default.insertLocal(LOCAL_DOC_KEY, { - // Set initial time to when the lineup simulator was released - time: timeInSecondsUTC(new Date('2022/10/27')), - }) - } - const lastSyncTime: number = localDoc.get('time') - console.info(`Last sync at ${new Date(lastSyncTime * 1000)}.`) - const currentTime = currentTimeSeconds() - await db.default.upsertLocal(LOCAL_DOC_KEY, { time: currentTime }) - const bar = new SingleBar({ - format: '\u2595{bar}\u258F {percentage}% | ETA: {eta}s | builds: {builds}', - barCompleteChar: '\u2588', - barIncompleteChar: '\u2591', - }) - let finishStatus = 'Reached the end.' - let lastCreatedAt = currentTime - let totalBuilds = 0 - bar.start(currentTime - lastSyncTime, 0, { - builds: 0, - }) - for await (const teams of fetchLineupSimulatorBuilds()) { - const createdAt = Number.parseInt(teams.created_at) - if (createdAt < lastSyncTime) { - finishStatus = 'Caught up with old data' - bar.update(bar.getTotal()) - break - } else if (createdAt > lastCreatedAt) { - finishStatus = 'Broke order' - break - } - bar.update(currentTime - createdAt, { - builds: totalBuilds, - }) - lastCreatedAt = createdAt - for (const team of teams.avatar_group) { - for (const character of team.group) { - totalBuilds += 1 - const substatKeys = character.secondary_attr_name.map((stat) => - getStatKey(stat.name) - ) - for (const artifactSet of character.set_list) { - const [sands, goblet, circlet] = character.first_attr - .sort((a, b) => a.cat_id - b.cat_id) - .map((s) => s.name) - const slots = { - flower: 'HP', - plume: 'ATK', - sands, - goblet, - circlet, - } - for (const [type, mainStat] of Object.entries(slots)) { - for (const substat of substatKeys) { - const row = { - set: getArtifactSet(artifactSet['name']), - slot: type, - main: getStatKey(mainStat), - sub: substat, - } - let doc = await db.default - .findOne({ - selector: row, - }) - .exec() - if (!doc) { - doc = await db.default.insert({ - ...row, - popularity: 0, - rarity: 0, - }) - } - await doc.update({ - $inc: { - popularity: 1, - }, - }) - } - } - } - } - } - } - bar.stop() - console.info(finishStatus) - await db.destroy() - process.exit() -} - -export const command = asCommand({ - command: 'popularity', - describe: - 'Scrapes character loadout data from the lineup simulator: https://act.hoyolab.com/ys/event/bbs-lineup-ys-sea/index.html', - builder: {}, - handler: () => { - createStatistics() - }, -}) diff --git a/packages/scripts/src/rarity.ts b/packages/scripts/src/rarity.ts deleted file mode 100644 index d32967e..0000000 --- a/packages/scripts/src/rarity.ts +++ /dev/null @@ -1,64 +0,0 @@ -import path from 'path' - -import { - mainStatRollChance, - subStatRollChance, - getDatabase, -} from '@gl/automation' -import { MainStatKey, SetKey, SlotKey, SubStatKey } from '@gl/types' - -import { asCommand } from './types' - -export async function insertRarity() { - // Ideally the database would be accessed in a way thats consistent independent of cwd - // Tried moving the db to the public dir, but it couldn't be accessed at all that way - process.chdir(path.resolve(__dirname, '../../..')) - - const db = await getDatabase() - - for (const set of Object.values(SetKey)) { - for (const slot of Object.values(SlotKey)) { - for (const main of Object.values(MainStatKey)) { - for (const sub of Object.values(SubStatKey)) { - const rarity = - mainStatRollChance(main, slot) * subStatRollChance(sub, main) - if (rarity === 0) { - // Rarity is zero if the artifact is not valid. - continue - } - const row = { - set, - slot, - main, - sub, - } - let doc = await db.default.findOne({ selector: row }).exec() - if (!doc) { - doc = await db.default.insert({ - ...row, - popularity: 0, - rarity: 0, - }) - } - await doc.update({ - $set: { - rarity, - }, - }) - } - } - } - } - - await db.destroy() - process.exit() -} - -export const command = asCommand({ - command: 'rarity', - describe: 'Hydrate the database with rarity information.', - builder: {}, - handler: () => { - insertRarity() - }, -}) diff --git a/packages/scripts/src/types.ts b/packages/scripts/src/types.ts deleted file mode 100644 index 31e1c83..0000000 --- a/packages/scripts/src/types.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { - ArgumentsCamelCase, - InferredOptionTypes, - Options, - CommandModule, -} from 'yargs' - -type OptionsDict = { [key: string]: Options } - -/** - * The original CommandModule type did not make the `command` field consistent with `handler` so here is a new type. - */ -interface MyCommandModule - extends CommandModule> { - builder?: O & OptionsDict - handler: ( - args: ArgumentsCamelCase> - ) => void | Promise -} - -/** - * Due to a limitation with typescript, we need this function to infer the generic portion of MyCommandModule. - */ -export const asCommand = ( - module: MyCommandModule -): CommandModule> => module diff --git a/packages/scripts/src/update-electron-vendors.mjs b/packages/scripts/src/update-electron-vendors.mjs deleted file mode 100644 index f7b8d55..0000000 --- a/packages/scripts/src/update-electron-vendors.mjs +++ /dev/null @@ -1,18 +0,0 @@ -/** - * This script should be run in electron context - * @example - * ELECTRON_RUN_AS_NODE=1 electron scripts/update-electron-vendors.mjs - */ - -import { writeFileSync } from 'fs' - -const electronRelease = process.versions - -const node = electronRelease.node.split('.')[0] -const chrome = electronRelease.v8.split('.').splice(0, 2).join('') - -writeFileSync( - '../../.electron-vendors.cache.json', - JSON.stringify({ chrome, node }) -) -writeFileSync('../../.browserslistrc', `Chrome ${chrome}`, 'utf8') diff --git a/packages/scripts/tsconfig.json b/packages/scripts/tsconfig.json index 3e68ae5..a16a6a8 100644 --- a/packages/scripts/tsconfig.json +++ b/packages/scripts/tsconfig.json @@ -3,5 +3,5 @@ "compilerOptions": { "noEmit": true }, - "include": ["src/**/*.ts"] + "include": ["bin/**/*.mjs"] } diff --git a/yarn.lock b/yarn.lock index cca7e23..d032c21 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2575,6 +2575,13 @@ dependencies: nx "15.8.9" +"@nrwl/devkit@18.0.4": + version "18.0.4" + resolved "https://registry.yarnpkg.com/@nrwl/devkit/-/devkit-18.0.4.tgz#5135f2f27218ca34d7b93f95f9967f75320ce75a" + integrity sha512-fKHnjg4/9MdFd2U4e8p6ja9fRa864DCyF70kB4YUB9NuUIgWLQ15Uj6wXC3xjdXmxQRyHDa7ORodVoFzdo4UCg== + dependencies: + "@nx/devkit" "18.0.4" + "@nrwl/devkit@>=15.5.2 < 16": version "15.8.9" resolved "https://registry.yarnpkg.com/@nrwl/devkit/-/devkit-15.8.9.tgz#556797672d0efa333c7af5ea252218f0815cecb8" @@ -2639,6 +2646,20 @@ dependencies: nx "15.8.9" +"@nx/devkit@18.0.4", "@nx/devkit@^18.0.4": + version "18.0.4" + resolved "https://registry.yarnpkg.com/@nx/devkit/-/devkit-18.0.4.tgz#fec28139ef5d078933c5f1b5a7e0a585672dbdd6" + integrity sha512-Vs1AXgOjMJyaWpKopD04dy0FwQ22n5ZR1bFf98Ab4Ht0WJwJE90IpUVAkwI03n5BYxAKOlQnFltsB4gu6Y8mZQ== + dependencies: + "@nrwl/devkit" "18.0.4" + ejs "^3.1.7" + enquirer "~2.3.6" + ignore "^5.0.4" + semver "^7.5.3" + tmp "~0.2.1" + tslib "^2.3.0" + yargs-parser "21.1.1" + "@octokit/auth-token@^3.0.0": version "3.0.3" resolved "https://registry.yarnpkg.com/@octokit/auth-token/-/auth-token-3.0.3.tgz#ce7e48a3166731f26068d7a7a7996b5da58cbe0c" @@ -16216,6 +16237,13 @@ semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +semver@^7.5.3: + version "7.6.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" + integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== + dependencies: + lru-cache "^6.0.0" + semver@~7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" From e5fb6a011ede005f8f7a6f91f79ded0d6aa39e47 Mon Sep 17 00:00:00 2001 From: Lucas Pollice Date: Sat, 17 Feb 2024 21:46:29 -0500 Subject: [PATCH 3/3] Revert "refactor scripts, remove dead code" This reverts commit 1be8f9f71af6cf2a1d77c011d2667fd760ea9ddc. --- package.json | 2 +- .../database/collections/default/index.ts | 1 + .../database/collections/default/v0.ts | 51 ++++ .../database/collections/default/v1.ts | 75 ++++++ .../database/collections/default/v2.ts | 97 ++++++++ .../src/scoring/database/collections/index.ts | 11 + .../src/scoring/database/collections/types.ts | 8 + .../src/scoring/database/database.ts | 35 +++ .../automation/src/scoring/database/index.ts | 1 + packages/automation/src/scoring/index.ts | 1 + packages/scripts/bin/index.mjs | 11 - packages/scripts/bin/types.mjs | 26 --- .../scripts/bin/update-electron-vendors.mjs | 34 --- packages/scripts/package.json | 16 +- packages/scripts/src/index.ts | 14 ++ packages/scripts/src/popularity.ts | 219 ++++++++++++++++++ packages/scripts/src/rarity.ts | 64 +++++ packages/scripts/src/types.ts | 26 +++ .../scripts/src/update-electron-vendors.mjs | 18 ++ packages/scripts/tsconfig.json | 2 +- yarn.lock | 28 --- 21 files changed, 634 insertions(+), 106 deletions(-) create mode 100644 packages/automation/src/scoring/database/collections/default/index.ts create mode 100644 packages/automation/src/scoring/database/collections/default/v0.ts create mode 100644 packages/automation/src/scoring/database/collections/default/v1.ts create mode 100644 packages/automation/src/scoring/database/collections/default/v2.ts create mode 100644 packages/automation/src/scoring/database/collections/index.ts create mode 100644 packages/automation/src/scoring/database/collections/types.ts create mode 100644 packages/automation/src/scoring/database/database.ts create mode 100644 packages/automation/src/scoring/database/index.ts delete mode 100644 packages/scripts/bin/index.mjs delete mode 100644 packages/scripts/bin/types.mjs delete mode 100644 packages/scripts/bin/update-electron-vendors.mjs create mode 100644 packages/scripts/src/index.ts create mode 100644 packages/scripts/src/popularity.ts create mode 100644 packages/scripts/src/rarity.ts create mode 100644 packages/scripts/src/types.ts create mode 100644 packages/scripts/src/update-electron-vendors.mjs diff --git a/package.json b/package.json index 3ec991e..4790848 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "compile:unpacked": "yarn build && electron-builder build --config .electron-builder.config.js --dir --config.asar=false", "watch": "node watch.mjs", "prepare": "husky install", - "postinstall": "cross-env ELECTRON_RUN_AS_NODE=1 electron packages/scripts/bin/index.mjs update-electron-vendors" + "postinstall": "lerna run set-env-targets --scope=@gl/scripts" }, "keywords": [], "author": { diff --git a/packages/automation/src/scoring/database/collections/default/index.ts b/packages/automation/src/scoring/database/collections/default/index.ts new file mode 100644 index 0000000..523b9ef --- /dev/null +++ b/packages/automation/src/scoring/database/collections/default/index.ts @@ -0,0 +1 @@ +export * from './v2' diff --git a/packages/automation/src/scoring/database/collections/default/v0.ts b/packages/automation/src/scoring/database/collections/default/v0.ts new file mode 100644 index 0000000..7bf38c0 --- /dev/null +++ b/packages/automation/src/scoring/database/collections/default/v0.ts @@ -0,0 +1,51 @@ +import { + RxJsonSchema, + toTypedRxJsonSchema, + ExtractDocumentTypeFromTypedRxJsonSchema, + MigrationStrategies, +} from 'rxdb' + +const schemaLiteral = { + title: 'precalculation schema', + description: 'precalculated scores', + version: 0, + primaryKey: { + key: 'id', + fields: ['set', 'slot', 'main', 'sub'], + separator: '|', + }, + type: 'object', + properties: { + id: { + type: 'string', + maxLength: 100, + }, + set: { + type: 'string', + }, + slot: { + type: 'string', + }, + main: { + type: 'string', + }, + sub: { + type: 'string', + }, + popularity: { + type: 'number', + default: 0, + }, + }, + required: ['set', 'slot', 'main', 'sub'], +} as const + +const schemaTyped = toTypedRxJsonSchema(schemaLiteral) + +export type RxDocType = ExtractDocumentTypeFromTypedRxJsonSchema< + typeof schemaTyped +> + +export const schema: RxJsonSchema = schemaTyped + +export const migrationStrategies: MigrationStrategies = {} diff --git a/packages/automation/src/scoring/database/collections/default/v1.ts b/packages/automation/src/scoring/database/collections/default/v1.ts new file mode 100644 index 0000000..3ac06fd --- /dev/null +++ b/packages/automation/src/scoring/database/collections/default/v1.ts @@ -0,0 +1,75 @@ +import { + RxJsonSchema, + toTypedRxJsonSchema, + ExtractDocumentTypeFromTypedRxJsonSchema, + MigrationStrategies, +} from 'rxdb' + +import { MigrationStrategy } from '../types' + +import { + RxDocType as LastRxDocType, + migrationStrategies as previousMigrationStrategies, +} from './v0' + +const VERSION = 1 + +const schemaLiteral = { + title: 'precalculation schema', + description: 'precalculated scores', + version: VERSION, + primaryKey: { + key: 'id', + fields: ['set', 'slot', 'main', 'sub'], + separator: '|', + }, + type: 'object', + properties: { + id: { + type: 'string', + maxLength: 100, + }, + set: { + type: 'string', + }, + slot: { + type: 'string', + }, + main: { + type: 'string', + }, + sub: { + type: 'string', + }, + popularity: { + type: 'number', + }, + rarity: { + type: 'number', + }, + }, + required: ['set', 'slot', 'main', 'sub', 'popularity', 'rarity'], +} as const + +const schemaTyped = toTypedRxJsonSchema(schemaLiteral) + +export type RxDocType = ExtractDocumentTypeFromTypedRxJsonSchema< + typeof schemaTyped +> + +export const schema: RxJsonSchema = schemaTyped + +const migrationStrategy: MigrationStrategy = ( + oldDoc +) => { + return { + ...oldDoc, + popularity: oldDoc.popularity ?? 0, + rarity: 0, + } +} + +export const migrationStrategies: MigrationStrategies = { + ...previousMigrationStrategies, + [VERSION]: migrationStrategy, +} diff --git a/packages/automation/src/scoring/database/collections/default/v2.ts b/packages/automation/src/scoring/database/collections/default/v2.ts new file mode 100644 index 0000000..2dabc75 --- /dev/null +++ b/packages/automation/src/scoring/database/collections/default/v2.ts @@ -0,0 +1,97 @@ +import { + RxJsonSchema, + toTypedRxJsonSchema, + ExtractDocumentTypeFromTypedRxJsonSchema, + MigrationStrategies, + RxCollectionCreator, +} from 'rxdb' + +import { MigrationStrategy } from '../types' + +import { + RxDocType as LastRxDocType, + migrationStrategies as previousMigrationStrategies, +} from './v1' + +const VERSION = 2 + +const scoreProperties = { + rarity: { + type: 'number', + minimum: 0, + maximum: 1, + }, + popularity: { + type: 'number', + minimum: 0, + multipleOf: 1, + }, +} as const + +export type CachedScores = keyof typeof scoreProperties +export const cachedScoreTypes = Object.keys(scoreProperties) as CachedScores[] + +const schemaLiteral = { + title: 'precalculation schema', + description: 'precalculated scores', + version: VERSION, + primaryKey: { + key: 'id', + fields: ['set', 'slot', 'main', 'sub'], + separator: '|', + }, + type: 'object', + properties: { + id: { + type: 'string', + maxLength: 100, + }, + set: { + type: 'string', + }, + slot: { + type: 'string', + }, + main: { + type: 'string', + }, + sub: { + type: 'string', + }, + ...scoreProperties, + }, + required: [ + 'set', + 'slot', + 'main', + 'sub', + ...(Object.keys(scoreProperties) as CachedScores[]), + ], +} as const // satisfies Parameters[0] + +const schemaTyped = toTypedRxJsonSchema(schemaLiteral) + +export type RxDocType = ExtractDocumentTypeFromTypedRxJsonSchema< + typeof schemaTyped +> + +export const schema: RxJsonSchema = schemaTyped + +const migrationStrategy: MigrationStrategy = ( + oldDoc +) => { + return { + ...oldDoc, + } +} + +export const migrationStrategies: MigrationStrategies = { + ...previousMigrationStrategies, + [VERSION]: migrationStrategy, +} + +export const collection: RxCollectionCreator = { + schema, + migrationStrategies, + localDocuments: true, +} diff --git a/packages/automation/src/scoring/database/collections/index.ts b/packages/automation/src/scoring/database/collections/index.ts new file mode 100644 index 0000000..f5dd34e --- /dev/null +++ b/packages/automation/src/scoring/database/collections/index.ts @@ -0,0 +1,11 @@ +import type { RxCollection, RxCollectionCreator } from 'rxdb' + +import * as Default from './default' + +export type Collections = { + default: RxCollection +} + +export const collections: Record = { + default: Default.collection, +} diff --git a/packages/automation/src/scoring/database/collections/types.ts b/packages/automation/src/scoring/database/collections/types.ts new file mode 100644 index 0000000..6a8eaff --- /dev/null +++ b/packages/automation/src/scoring/database/collections/types.ts @@ -0,0 +1,8 @@ +import { OldRxCollection } from 'rxdb' + +type MaybePromise = T | Promise + +export type MigrationStrategy = ( + oldDocumentData: OldDocData, + oldRxCollection: OldRxCollection +) => MaybePromise diff --git a/packages/automation/src/scoring/database/database.ts b/packages/automation/src/scoring/database/database.ts new file mode 100644 index 0000000..5dace71 --- /dev/null +++ b/packages/automation/src/scoring/database/database.ts @@ -0,0 +1,35 @@ +import { LokiFsAdapter, LokiMemoryAdapter } from 'lokijs' +import { createRxDatabase, addRxPlugin, RxDatabase } from 'rxdb' +import { RxDBDevModePlugin } from 'rxdb/plugins/dev-mode' +import { RxDBJsonDumpPlugin } from 'rxdb/plugins/json-dump' +import { RxDBLocalDocumentsPlugin } from 'rxdb/plugins/local-documents' +import { getRxStorageLoki } from 'rxdb/plugins/lokijs' +import { RxDBMigrationPlugin } from 'rxdb/plugins/migration' +import { RxDBQueryBuilderPlugin } from 'rxdb/plugins/query-builder' +import { RxDBUpdatePlugin } from 'rxdb/plugins/update' + +import { Collections, collections } from './collections' + +addRxPlugin(RxDBDevModePlugin) +addRxPlugin(RxDBJsonDumpPlugin) +addRxPlugin(RxDBLocalDocumentsPlugin) +addRxPlugin(RxDBMigrationPlugin) +addRxPlugin(RxDBQueryBuilderPlugin) +addRxPlugin(RxDBUpdatePlugin) + +let db: Promise> +export async function getDatabase(importData = true) { + if (db) { + return db + } + db = createRxDatabase({ + name: 'default', + storage: getRxStorageLoki({ + adapter: importData ? new LokiFsAdapter() : new LokiMemoryAdapter(), + }), + }).then(async (newDB) => { + await newDB.addCollections(collections) + return newDB + }) + return db +} diff --git a/packages/automation/src/scoring/database/index.ts b/packages/automation/src/scoring/database/index.ts new file mode 100644 index 0000000..e1ddd7b --- /dev/null +++ b/packages/automation/src/scoring/database/index.ts @@ -0,0 +1 @@ +export * from './database' diff --git a/packages/automation/src/scoring/index.ts b/packages/automation/src/scoring/index.ts index abf400f..12226cd 100644 --- a/packages/automation/src/scoring/index.ts +++ b/packages/automation/src/scoring/index.ts @@ -1,3 +1,4 @@ export * from './logic' export * from './types' +export { getDatabase } from './database' export * from './const' diff --git a/packages/scripts/bin/index.mjs b/packages/scripts/bin/index.mjs deleted file mode 100644 index ff98c61..0000000 --- a/packages/scripts/bin/index.mjs +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env node -import { hideBin } from 'yargs/helpers' -import yargs from 'yargs/yargs' - -import * as updateElectronVendors from './update-electron-vendors.mjs' - -yargs(hideBin(process.argv)) - .command(updateElectronVendors.command) - .demandCommand() - .help() - .parse() diff --git a/packages/scripts/bin/types.mjs b/packages/scripts/bin/types.mjs deleted file mode 100644 index 391f40e..0000000 --- a/packages/scripts/bin/types.mjs +++ /dev/null @@ -1,26 +0,0 @@ -/** - * @typedef {import("yargs").Options} Options - * @typedef {import("yargs").ArgumentsCamelCase} ArgumentsCamelCase - * @typedef {{ [key: string]: Options; }} OptionsDict - */ - -/** - * @template {OptionsDict} O - * @typedef {import("yargs").CommandModule>} CommandModule - */ - -/** - * @template {OptionsDict} O - * @typedef {CommandModule & { - * builder?: O & OptionsDict; - * handler: (args: import("yargs").ArgumentsCamelCase>) => void | Promise; - * }} MyCommandModule - */ - -/** - * Due to a limitation with typescript, we need this function to infer the generic portion of MyCommandModule. - * @template {OptionsDict} T - * @param {MyCommandModule} module - * @returns {import("yargs").CommandModule>} - */ -export const asCommand = (module) => module diff --git a/packages/scripts/bin/update-electron-vendors.mjs b/packages/scripts/bin/update-electron-vendors.mjs deleted file mode 100644 index 0499895..0000000 --- a/packages/scripts/bin/update-electron-vendors.mjs +++ /dev/null @@ -1,34 +0,0 @@ -import { writeFileSync } from 'fs' -import path from 'path' - -import { workspaceRoot } from '@nx/devkit' - -import { asCommand } from './types.mjs' - -const electronRelease = process.versions - -const node = electronRelease.node.split('.')[0] -const chrome = electronRelease.v8.split('.').splice(0, 2).join('') - -export const command = asCommand({ - command: 'update-electron-vendors', - describe: - 'Generate the node and chrome targets for the current version of electron.', - builder: {}, - handler: () => { - if (!process.versions['electron']) { - throw new Error( - `This script should be run in electron context. Example: \`ELECTRON_RUN_AS_NODE=1 electron update-electron-vendors.mjs\`` - ) - } - writeFileSync( - path.join(workspaceRoot, '.electron-vendors.cache.json'), - JSON.stringify({ chrome, node }) - ) - writeFileSync( - path.join(workspaceRoot, '.browserslistrc'), - `Chrome ${chrome}`, - 'utf8' - ) - }, -}) diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 6187346..6fc81a4 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -5,7 +5,13 @@ "author": "Lucas Pollice ", "homepage": "https://github.com/Jugbot/genshin-locker#readme", "license": "ISC", - "bin": "bin/index.mjs", + "main": "src/index.ts", + "directories": { + "src": "src" + }, + "files": [ + "src" + ], "repository": { "type": "git", "url": "git+https://github.com/Jugbot/genshin-locker.git" @@ -16,13 +22,13 @@ "lint:write": "eslint . --max-warnings=0 --ignore-path=\"../../.gitignore\" --fix", "lint": "eslint . --max-warnings=0 --ignore-path=\"../../.gitignore\"", "check-types": "tsc --noEmit", - "scripts": "scripts" + "script": "vite-node --script src", + "set-env-targets": "cross-env ELECTRON_RUN_AS_NODE=1 electron src/update-electron-vendors.mjs" }, "bugs": { "url": "https://github.com/Jugbot/genshin-locker/issues" }, "dependencies": { - "@nx/devkit": "^18.0.4", "@gl/automation": "*", "@gl/types": "*", "cli-progress": "^3.11.2", @@ -36,7 +42,7 @@ "electron": "^24.0.0", "ts-node": "^10.9.1", "typescript": "^4.9.0", - "vite": "4.2.1", - "vite-node": "^0.33.0" + "vite-node": "^0.33.0", + "vite": "4.2.1" } } diff --git a/packages/scripts/src/index.ts b/packages/scripts/src/index.ts new file mode 100644 index 0000000..877aecd --- /dev/null +++ b/packages/scripts/src/index.ts @@ -0,0 +1,14 @@ +import { hideBin } from 'yargs/helpers' +import yargs from 'yargs/yargs' + +import * as popularity from './popularity' +import * as rarity from './rarity' +// import * as vendors from './update-electron-vendors' + +yargs(hideBin(process.argv)) + .command(rarity.command) + .command(popularity.command) + // .command(vendors.command) + .demandCommand() + .help() + .parse() diff --git a/packages/scripts/src/popularity.ts b/packages/scripts/src/popularity.ts new file mode 100644 index 0000000..ad5c33d --- /dev/null +++ b/packages/scripts/src/popularity.ts @@ -0,0 +1,219 @@ +import { getDatabase, getArtifactSet, getStatKey } from '@gl/automation' +import { SingleBar } from 'cli-progress' +import fetch from 'node-fetch' + +import { asCommand } from './types' + +const timeInSecondsUTC = (date: Date) => { + return Math.trunc(date.getTime() / 1000) +} +const currentTimeSeconds = () => { + return timeInSecondsUTC(new Date()) +} +// TODO: scrape data from other languages +type Payload = { + data: { + next_page_token: string + list: Array<{ + id: string + account_uid: string + nickname: string + avatar_url: string + level: number + title: string + tag_ids: Array + avatar_group: Array<{ + group: Array<{ + id: number + name: string + icon: string + element: number + level: number + weapon_cat_id: number + weapon: { + id: number + name: string + icon: string + level: number + cat_id: number + wiki_url: string + } + set_list: Array<{ + id: number + name: string + icon: string + level: number + attr_id: number + cat_id: number + wiki_url: string + }> + wiki_url: string + avatar_tag: { + id: number + name: string + color: string + } + first_attr: Array<{ + cat_id: number + id: number + name: string + }> + secondary_attr: [] + secondary_attr_name: Array<{ + id: number + name: string + }> + head_icon: string + pc_icon: string + }> + }> + quick_avatar: [] + description: string + like_cnt: number + comment_cnt: number + favour_cnt: number + view_cnt: number + is_favour: boolean + is_like: boolean + status: number + is_deleted: boolean + created_at: string + group_cnt: number + trans: boolean + origin_title: string + origin_desc: string + tag_name: [] + trans_from: string + }> + } +} + +async function* fetchLineupSimulatorBuilds(limit = 1000) { + let nextPageToken = '' + while (nextPageToken !== null) { + const request = new URL( + 'https://sg-public-api.hoyolab.com/event/simulatoros/lineup/index' + ) + request.searchParams.set('limit', `${limit}`) + request.searchParams.set('next_page_token', nextPageToken) + request.searchParams.set('tag_id', '2') + request.searchParams.set('roles', '') + request.searchParams.set('order', 'CreatedTime') + request.searchParams.set('lang', 'en-us') + try { + const data = await fetch(request) + const jsonData = (await data.json()) as Payload + nextPageToken = jsonData.data['next_page_token'] + yield* jsonData.data['list'] + } catch (e) { + console.error(e) + } + } +} + +const LOCAL_DOC_KEY = 'sync-time' +type LocalDoc = { + time: number +} + +async function createStatistics() { + const db = await getDatabase() + let localDoc = await db.default.getLocal(LOCAL_DOC_KEY) + if (localDoc === null) { + localDoc = await db.default.insertLocal(LOCAL_DOC_KEY, { + // Set initial time to when the lineup simulator was released + time: timeInSecondsUTC(new Date('2022/10/27')), + }) + } + const lastSyncTime: number = localDoc.get('time') + console.info(`Last sync at ${new Date(lastSyncTime * 1000)}.`) + const currentTime = currentTimeSeconds() + await db.default.upsertLocal(LOCAL_DOC_KEY, { time: currentTime }) + const bar = new SingleBar({ + format: '\u2595{bar}\u258F {percentage}% | ETA: {eta}s | builds: {builds}', + barCompleteChar: '\u2588', + barIncompleteChar: '\u2591', + }) + let finishStatus = 'Reached the end.' + let lastCreatedAt = currentTime + let totalBuilds = 0 + bar.start(currentTime - lastSyncTime, 0, { + builds: 0, + }) + for await (const teams of fetchLineupSimulatorBuilds()) { + const createdAt = Number.parseInt(teams.created_at) + if (createdAt < lastSyncTime) { + finishStatus = 'Caught up with old data' + bar.update(bar.getTotal()) + break + } else if (createdAt > lastCreatedAt) { + finishStatus = 'Broke order' + break + } + bar.update(currentTime - createdAt, { + builds: totalBuilds, + }) + lastCreatedAt = createdAt + for (const team of teams.avatar_group) { + for (const character of team.group) { + totalBuilds += 1 + const substatKeys = character.secondary_attr_name.map((stat) => + getStatKey(stat.name) + ) + for (const artifactSet of character.set_list) { + const [sands, goblet, circlet] = character.first_attr + .sort((a, b) => a.cat_id - b.cat_id) + .map((s) => s.name) + const slots = { + flower: 'HP', + plume: 'ATK', + sands, + goblet, + circlet, + } + for (const [type, mainStat] of Object.entries(slots)) { + for (const substat of substatKeys) { + const row = { + set: getArtifactSet(artifactSet['name']), + slot: type, + main: getStatKey(mainStat), + sub: substat, + } + let doc = await db.default + .findOne({ + selector: row, + }) + .exec() + if (!doc) { + doc = await db.default.insert({ + ...row, + popularity: 0, + rarity: 0, + }) + } + await doc.update({ + $inc: { + popularity: 1, + }, + }) + } + } + } + } + } + } + bar.stop() + console.info(finishStatus) + await db.destroy() + process.exit() +} + +export const command = asCommand({ + command: 'popularity', + describe: + 'Scrapes character loadout data from the lineup simulator: https://act.hoyolab.com/ys/event/bbs-lineup-ys-sea/index.html', + builder: {}, + handler: () => { + createStatistics() + }, +}) diff --git a/packages/scripts/src/rarity.ts b/packages/scripts/src/rarity.ts new file mode 100644 index 0000000..d32967e --- /dev/null +++ b/packages/scripts/src/rarity.ts @@ -0,0 +1,64 @@ +import path from 'path' + +import { + mainStatRollChance, + subStatRollChance, + getDatabase, +} from '@gl/automation' +import { MainStatKey, SetKey, SlotKey, SubStatKey } from '@gl/types' + +import { asCommand } from './types' + +export async function insertRarity() { + // Ideally the database would be accessed in a way thats consistent independent of cwd + // Tried moving the db to the public dir, but it couldn't be accessed at all that way + process.chdir(path.resolve(__dirname, '../../..')) + + const db = await getDatabase() + + for (const set of Object.values(SetKey)) { + for (const slot of Object.values(SlotKey)) { + for (const main of Object.values(MainStatKey)) { + for (const sub of Object.values(SubStatKey)) { + const rarity = + mainStatRollChance(main, slot) * subStatRollChance(sub, main) + if (rarity === 0) { + // Rarity is zero if the artifact is not valid. + continue + } + const row = { + set, + slot, + main, + sub, + } + let doc = await db.default.findOne({ selector: row }).exec() + if (!doc) { + doc = await db.default.insert({ + ...row, + popularity: 0, + rarity: 0, + }) + } + await doc.update({ + $set: { + rarity, + }, + }) + } + } + } + } + + await db.destroy() + process.exit() +} + +export const command = asCommand({ + command: 'rarity', + describe: 'Hydrate the database with rarity information.', + builder: {}, + handler: () => { + insertRarity() + }, +}) diff --git a/packages/scripts/src/types.ts b/packages/scripts/src/types.ts new file mode 100644 index 0000000..31e1c83 --- /dev/null +++ b/packages/scripts/src/types.ts @@ -0,0 +1,26 @@ +import { + ArgumentsCamelCase, + InferredOptionTypes, + Options, + CommandModule, +} from 'yargs' + +type OptionsDict = { [key: string]: Options } + +/** + * The original CommandModule type did not make the `command` field consistent with `handler` so here is a new type. + */ +interface MyCommandModule + extends CommandModule> { + builder?: O & OptionsDict + handler: ( + args: ArgumentsCamelCase> + ) => void | Promise +} + +/** + * Due to a limitation with typescript, we need this function to infer the generic portion of MyCommandModule. + */ +export const asCommand = ( + module: MyCommandModule +): CommandModule> => module diff --git a/packages/scripts/src/update-electron-vendors.mjs b/packages/scripts/src/update-electron-vendors.mjs new file mode 100644 index 0000000..f7b8d55 --- /dev/null +++ b/packages/scripts/src/update-electron-vendors.mjs @@ -0,0 +1,18 @@ +/** + * This script should be run in electron context + * @example + * ELECTRON_RUN_AS_NODE=1 electron scripts/update-electron-vendors.mjs + */ + +import { writeFileSync } from 'fs' + +const electronRelease = process.versions + +const node = electronRelease.node.split('.')[0] +const chrome = electronRelease.v8.split('.').splice(0, 2).join('') + +writeFileSync( + '../../.electron-vendors.cache.json', + JSON.stringify({ chrome, node }) +) +writeFileSync('../../.browserslistrc', `Chrome ${chrome}`, 'utf8') diff --git a/packages/scripts/tsconfig.json b/packages/scripts/tsconfig.json index a16a6a8..3e68ae5 100644 --- a/packages/scripts/tsconfig.json +++ b/packages/scripts/tsconfig.json @@ -3,5 +3,5 @@ "compilerOptions": { "noEmit": true }, - "include": ["bin/**/*.mjs"] + "include": ["src/**/*.ts"] } diff --git a/yarn.lock b/yarn.lock index d032c21..cca7e23 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2575,13 +2575,6 @@ dependencies: nx "15.8.9" -"@nrwl/devkit@18.0.4": - version "18.0.4" - resolved "https://registry.yarnpkg.com/@nrwl/devkit/-/devkit-18.0.4.tgz#5135f2f27218ca34d7b93f95f9967f75320ce75a" - integrity sha512-fKHnjg4/9MdFd2U4e8p6ja9fRa864DCyF70kB4YUB9NuUIgWLQ15Uj6wXC3xjdXmxQRyHDa7ORodVoFzdo4UCg== - dependencies: - "@nx/devkit" "18.0.4" - "@nrwl/devkit@>=15.5.2 < 16": version "15.8.9" resolved "https://registry.yarnpkg.com/@nrwl/devkit/-/devkit-15.8.9.tgz#556797672d0efa333c7af5ea252218f0815cecb8" @@ -2646,20 +2639,6 @@ dependencies: nx "15.8.9" -"@nx/devkit@18.0.4", "@nx/devkit@^18.0.4": - version "18.0.4" - resolved "https://registry.yarnpkg.com/@nx/devkit/-/devkit-18.0.4.tgz#fec28139ef5d078933c5f1b5a7e0a585672dbdd6" - integrity sha512-Vs1AXgOjMJyaWpKopD04dy0FwQ22n5ZR1bFf98Ab4Ht0WJwJE90IpUVAkwI03n5BYxAKOlQnFltsB4gu6Y8mZQ== - dependencies: - "@nrwl/devkit" "18.0.4" - ejs "^3.1.7" - enquirer "~2.3.6" - ignore "^5.0.4" - semver "^7.5.3" - tmp "~0.2.1" - tslib "^2.3.0" - yargs-parser "21.1.1" - "@octokit/auth-token@^3.0.0": version "3.0.3" resolved "https://registry.yarnpkg.com/@octokit/auth-token/-/auth-token-3.0.3.tgz#ce7e48a3166731f26068d7a7a7996b5da58cbe0c" @@ -16237,13 +16216,6 @@ semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== -semver@^7.5.3: - version "7.6.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" - integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== - dependencies: - lru-cache "^6.0.0" - semver@~7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e"