From a5fc6149e0e9ea26da97199b9a8eb034f4f68112 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 6 Nov 2024 17:30:09 +0400 Subject: [PATCH 01/19] Use another ts proto API for griffin UI & tracker --- src/core/serializers.ts | 18 +++---- src/core/study_processor.ts | 47 +++++++++---------- src/core/summary.ts | 6 +-- src/finch_tracker/main.ts | 6 +-- src/finch_tracker/tracker_lib.test.ts | 15 +++--- src/finch_tracker/tracker_lib.ts | 7 +-- src/scripts/generate_proto_ts.ts | 34 ++------------ src/seed_tools/utils/seed_validation.test.ts | 10 ++-- src/seed_tools/utils/study_json_utils.test.ts | 6 +-- src/seed_tools/utils/study_validation.test.ts | 16 +++---- src/web/app/app.tsx | 10 ++-- src/web/app/experiment_model.ts | 8 ++-- src/web/app/seed_loader.ts | 4 +- src/web/app/study_model.test.ts | 23 +++++---- src/web/app/study_model.ts | 10 ++-- 15 files changed, 101 insertions(+), 119 deletions(-) diff --git a/src/core/serializers.ts b/src/core/serializers.ts index 034c5db1..01cdd095 100644 --- a/src/core/serializers.ts +++ b/src/core/serializers.ts @@ -2,7 +2,7 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -import { variations as proto } from '../proto/generated/proto_bundle'; +import {Study, Study_Channel, Study_Platform} from '../proto/generated/study'; export function getPlatformNameFromString(protoPlatfrom: string): string { const PREFIX = 'PLATFORM_'; @@ -11,8 +11,8 @@ export function getPlatformNameFromString(protoPlatfrom: string): string { return protoPlatfrom; } -export function getPlatfromName(protoPlatfrom: proto.Study.Platform): string { - return getPlatformNameFromString(proto.Study.Platform[protoPlatfrom]); +export function getPlatfromName(protoPlatfrom: Study_Platform): string { + return getPlatformNameFromString(Study_Platform[protoPlatfrom]); } export function getChannelNameFromString( @@ -27,11 +27,11 @@ export function getChannelNameFromString( } export function getChannelName( - protoChannel: proto.Study.Channel, + protoChannel: Study_Channel, isBraveSpecific: boolean, ): string { return getChannelNameFromString( - proto.Study.Channel[protoChannel], + Study_Channel[protoChannel], isBraveSpecific, ); } @@ -52,9 +52,11 @@ export function serializeChannels(channels?: string[]): string | undefined { // Converts a study to JSON that is ready to be serialized. Some field are // removed, some are converted to a human readable format. -export function studyToJSON(study: proto.IStudy): Record { - const msg = proto.Study.fromObject(study); - const json = msg.toJSON(); +export function studyToJSON(study: Study): Record { + const json = Study.toJson(study) as Record | null; + if (json === null) { + throw new Error('Failed to convert study to JSON'); + } const filter = json.filter; delete json.consistency; delete json.activation_type; diff --git a/src/core/study_processor.ts b/src/core/study_processor.ts index 77d2039b..36732fed 100644 --- a/src/core/study_processor.ts +++ b/src/core/study_processor.ts @@ -2,20 +2,20 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -import { variations as proto } from '../proto/generated/proto_bundle'; +import { Study, Study_Channel, Study_Experiment, Study_Filter, Study_Platform} from '../proto/generated/study'; import { type ProcessingOptions } from './base_types'; import { isFeatureBlocklisted, isStudyNameBlocklisted } from './blocklists'; import { matchesMaxVersion, parseVersionPattern } from './version'; -const UPSTREAM_SUPPORTED_PLATFORMS: readonly proto.Study.Platform[] = [ - proto.Study.Platform.PLATFORM_ANDROID, - proto.Study.Platform.PLATFORM_LINUX, - proto.Study.Platform.PLATFORM_MAC, - proto.Study.Platform.PLATFORM_WINDOWS, +const UPSTREAM_SUPPORTED_PLATFORMS: readonly Study_Platform[] = [ + Study_Platform.ANDROID, + Study_Platform.LINUX, + Study_Platform.MAC, + Study_Platform.WINDOWS, ]; -const BRAVE_SUPPORTED_PLATFORMS: readonly proto.Study.Platform[] = - UPSTREAM_SUPPORTED_PLATFORMS.concat([proto.Study.Platform.PLATFORM_IOS]); +const BRAVE_SUPPORTED_PLATFORMS: readonly Study_Platform[] = + UPSTREAM_SUPPORTED_PLATFORMS.concat([Study_Platform.IOS]); export enum StudyChannelTarget { // filter.channel includes DEV or CANNERY, doesn't include STABLE or BETA. @@ -86,14 +86,14 @@ export class StudyFilter { } } -// A wrapper over a raw proto.Study that processes it and collects some extra +// A wrapper over a raw Study that processes it and collects some extra // data. export class ProcessedStudy { - study: proto.IStudy; + study: Study; studyDetails: StudyDetails; affectedFeatures: Set; - constructor(study: proto.IStudy, options: ProcessingOptions) { + constructor(study: Study, options: ProcessingOptions) { this.study = study; this.studyDetails = new StudyDetails(study, options); this.affectedFeatures = getAffectedFeatures(study); @@ -106,7 +106,7 @@ export class ProcessedStudy { stripEmptyFilterGroups(): void { this.study.experiment = this.study.experiment?.filter( - (e) => e.probability_weight > 0, + (e) => (e.probability_weight ?? 0) > 0, ); } @@ -172,7 +172,7 @@ export class StudyDetails { maxNonDefaultIndex = -1; channelTarget = StudyChannelTarget.DEV_OR_CANARY; - constructor(study: proto.IStudy, options: ProcessingOptions) { + constructor(study: Study, options: ProcessingOptions) { const filter = study.filter; const experiment = study.experiment; const maxVersion = filter?.max_version; @@ -202,7 +202,7 @@ export class StudyDetails { endDateSeconds = filter.end_date; } else { // Long - endDateSeconds = filter.end_date.toNumber(); + endDateSeconds = Number(filter.end_date); } if ( @@ -227,7 +227,7 @@ export class StudyDetails { this.isBlocklisted ||= disabledFeatures?.some((n) => isFeatureBlocklisted(n)) ?? false; - this.isKillSwitch ||= e.probability_weight > 0 && isKillSwitch(e.name); + this.isKillSwitch ||= (e.probability_weight ?? 0) > 0 && isKillSwitch(e.name); this.onlyDisabledFeatures &&= e.probability_weight === 0 || @@ -242,7 +242,7 @@ export class StudyDetails { let index = 0; for (const e of experiment) { - const weight = e.probability_weight; + const weight = e.probability_weight ?? 0; this.totalWeight += weight; if ( !e.name.startsWith('Default') && @@ -259,9 +259,9 @@ export class StudyDetails { } const channel = study.filter?.channel; - if (channel?.includes(proto.Study.Channel.BETA)) + if (channel?.includes(Study_Channel.BETA)) this.channelTarget = StudyChannelTarget.BETA; - if (channel?.includes(proto.Study.Channel.STABLE)) + if (channel?.includes(Study_Channel.STABLE)) this.channelTarget = StudyChannelTarget.STABLE; } @@ -293,7 +293,7 @@ export class StudyDetails { } } -function getAffectedFeatures(study: proto.IStudy): Set { +function getAffectedFeatures(study: Study): Set { const features = new Set(); const experiment = study.experiment; if (experiment == null) { @@ -306,7 +306,7 @@ function getAffectedFeatures(study: proto.IStudy): Set { return features; } -function areFeaturesInDefaultStates(e: proto.Study.IExperiment): boolean { +function areFeaturesInDefaultStates(e: Study_Experiment): boolean { const enableFeature = e.feature_association?.enable_feature; const disableFeature = e.feature_association?.disable_feature; if (enableFeature != null && enableFeature.length > 0) return false; @@ -315,11 +315,10 @@ function areFeaturesInDefaultStates(e: proto.Study.IExperiment): boolean { } function filterPlatforms( - f: proto.Study.IFilter | undefined | null, + f: Study_Filter, isBraveSeed: boolean, -): proto.Study.Platform[] | undefined { +): Study_Platform[] { const platform = f?.platform; - if (platform == null) return undefined; const supportedPlatforms = isBraveSeed ? BRAVE_SUPPORTED_PLATFORMS : UPSTREAM_SUPPORTED_PLATFORMS; @@ -328,7 +327,7 @@ function filterPlatforms( // Processes a list of studies and groups it according to study.name. export function processStudyList( - list: proto.IStudy[], + list: Study[], minPriority: StudyPriority, options: ProcessingOptions, ): Map { diff --git a/src/core/summary.ts b/src/core/summary.ts index e403c260..117e2491 100644 --- a/src/core/summary.ts +++ b/src/core/summary.ts @@ -4,7 +4,6 @@ // You can obtain one at https://mozilla.org/MPL/2.0/. import { createHash } from 'node:crypto'; -import { type variations as proto } from '../proto/generated/proto_bundle'; import { SeedType, type ProcessingOptions } from './base_types'; import { StudyPriority, @@ -15,6 +14,7 @@ import { import * as config from '../config'; import * as url_utils from './url_utils'; +import { VariationsSeed } from '../proto/generated/variations_seed'; export enum ItemAction { New, @@ -115,8 +115,8 @@ function getOverallAudience( } export function makeSummary( - oldSeed: proto.VariationsSeed, - newSeed: proto.VariationsSeed, + oldSeed: VariationsSeed, + newSeed: VariationsSeed, options: ProcessingOptions, minPriority: StudyPriority, ): Map { diff --git a/src/finch_tracker/main.ts b/src/finch_tracker/main.ts index 9eba43d2..4c360881 100644 --- a/src/finch_tracker/main.ts +++ b/src/finch_tracker/main.ts @@ -8,13 +8,13 @@ import { type ProcessingOptions } from '../core/base_types'; import { StudyPriority } from '../core/study_processor'; import { makeSummary, summaryToJson } from '../core/summary'; import * as url_utils from '../core/url_utils'; -import { variations as proto } from '../proto/generated/proto_bundle'; import { downloadUrl, getSeedPath } from './node_utils'; import { commitAllChanges, fetchChromeSeedData, storeDataToDirectory, } from './tracker_lib'; +import { VariationsSeed } from '../proto/generated/variations_seed'; async function main(): Promise { const program = new Command() @@ -86,7 +86,7 @@ async function main(): Promise { seedFile !== undefined ? fs.readFileSync(seedFile) : await fetchChromeSeedData(); - const seed = proto.VariationsSeed.decode(seedData); + const seed = VariationsSeed.fromBinary(seedData); let previousSeedData: Buffer | undefined; let newGitSha1: string | undefined; @@ -105,7 +105,7 @@ async function main(): Promise { } if (createSummary && previousSeedData !== undefined) { - const previousSeed = proto.VariationsSeed.decode(previousSeedData); + const previousSeed = VariationsSeed.fromBinary(previousSeedData); const summary = makeSummary( previousSeed, seed, diff --git a/src/finch_tracker/tracker_lib.test.ts b/src/finch_tracker/tracker_lib.test.ts index 9a9d6a95..4e7c8278 100644 --- a/src/finch_tracker/tracker_lib.test.ts +++ b/src/finch_tracker/tracker_lib.test.ts @@ -8,7 +8,8 @@ import * as fs from 'fs'; import { describe, expect, test } from '@jest/globals'; import { StudyPriority } from '../core/study_processor'; import { ItemAction, makeSummary, summaryToJson } from '../core/summary'; -import { variations as proto } from '../proto/generated/proto_bundle'; +import { Study, Study_Channel, Study_Platform} from '../proto/generated/study'; +import { VariationsSeed } from '../proto/generated/variations_seed'; import { serializeStudies } from './tracker_lib'; function serialize(json: Record) { @@ -46,11 +47,11 @@ describe('summary', () => { const common = { name: 'TestStudy', filter: { - channel: [proto.Study.Channel.STABLE], - platform: [proto.Study.Platform.PLATFORM_WINDOWS], + channel: [Study_Channel.STABLE], + platform: [Study_Platform.WINDOWS], }, }; - const oldStudy = new proto.Study({ + const oldStudy = Study.create({ ...common, experiment: [ { @@ -65,7 +66,7 @@ describe('summary', () => { ], }); - const newStudy = new proto.Study({ + const newStudy = Study.create({ ...common, experiment: [ { @@ -84,8 +85,8 @@ describe('summary', () => { }, ], }); - const oldSeed = new proto.VariationsSeed({ study: [oldStudy] }); - const newSeed = new proto.VariationsSeed({ study: [newStudy] }); + const oldSeed = VariationsSeed.create({ study: [oldStudy] }); + const newSeed = VariationsSeed.create({ study: [newStudy] }); const summary = makeSummary( oldSeed, diff --git a/src/finch_tracker/tracker_lib.ts b/src/finch_tracker/tracker_lib.ts index f8af09be..7c1149a6 100644 --- a/src/finch_tracker/tracker_lib.ts +++ b/src/finch_tracker/tracker_lib.ts @@ -14,7 +14,8 @@ import { StudyPriority, priorityToText, } from '../core/study_processor'; -import { variations as proto } from '../proto/generated/proto_bundle'; +import { Study } from '../proto/generated/study'; +import { VariationsSeed } from '../proto/generated/variations_seed'; import { downloadUrl, getSeedPath, getStudyPath } from './node_utils'; export async function fetchChromeSeedData(): Promise { @@ -29,8 +30,8 @@ export function serializeStudies( options: ProcessingOptions, ): Record { const map: Record = {}; - const seed = proto.VariationsSeed.decode(seedData); - const addStudy = (path: string, study: proto.IStudy) => { + const seed = VariationsSeed.fromBinary(seedData); + const addStudy = (path: string, study: Study) => { const json = studyToJSON(study); const list = map[path]; if (list !== undefined) list.push(json); diff --git a/src/scripts/generate_proto_ts.ts b/src/scripts/generate_proto_ts.ts index a1c541c5..9d8408ad 100644 --- a/src/scripts/generate_proto_ts.ts +++ b/src/scripts/generate_proto_ts.ts @@ -36,7 +36,6 @@ function main(options: Options) { } removeGeneratedFiles(); - generateProtobufJsWithTypeInfo(); generateProtobufTs(); } @@ -49,33 +48,6 @@ function removeGeneratedFiles() { }); } -function generateProtobufJsWithTypeInfo() { - execSync( - [ - 'npx', - '--', - 'pbjs', - '--t', - 'static-module', - '--keep-case', - ...protoFiles, - '-o', - `${protoGeneratedDir}/proto_bundle.js`, - ].join(' '), - ); - - execSync( - [ - 'npx', - '--', - 'pbts', - '-o', - `${protoGeneratedDir}/proto_bundle.d.ts`, - `${protoGeneratedDir}/proto_bundle.js`, - ].join(' '), - ); -} - function generateProtobufTs() { try { // Apply study.proto patch to make protobuf-ts serialize probability_weight @@ -102,7 +74,7 @@ function generateProtobufTs() { function generateStudyProtoPatch() { fs.writeFileSync( - `${protoDir}/study.proto.protobuf-ts.patch`, + `${protoDir}/study.protobuf-ts.patch`, execSync(`git diff ${protoDir}/study.proto`, { encoding: 'buffer', }), @@ -110,14 +82,14 @@ function generateStudyProtoPatch() { } function gitApplyStudyProtoPatch() { - execSync(`git apply ${protoDir}/study.proto.protobuf-ts.patch`, { + execSync(`git apply ${protoDir}/study.protobuf-ts.patch`, { cwd: protoDir, stdio: 'inherit', }); } function gitRevertStudyProtoPatch() { - execSync(`git apply -R ${protoDir}/study.proto.protobuf-ts.patch`, { + execSync(`git apply -R ${protoDir}/study.protobuf-ts.patch`, { cwd: protoDir, stdio: 'inherit', }); diff --git a/src/seed_tools/utils/seed_validation.test.ts b/src/seed_tools/utils/seed_validation.test.ts index 3a4ef2fb..ff215233 100644 --- a/src/seed_tools/utils/seed_validation.test.ts +++ b/src/seed_tools/utils/seed_validation.test.ts @@ -31,7 +31,7 @@ describe('getSeedErrors', () => { } else { study.filter = {}; } - study.filter.platform = study.filter.platform ?? ['PLATFORM_LINUX']; + study.filter.platform = study.filter.platform ?? ['LINUX']; study.filter.channel = study.filter.channel ?? ['BETA']; return Study.fromJson(study, { ignoreUnknownFields: false, @@ -184,19 +184,19 @@ describe('getSeedErrors', () => { // platform tests { filter1: { - platform: ['PLATFORM_WINDOWS', 'PLATFORM_MAC'], + platform: ['WINDOWS', 'MAC'], }, filter2: { - platform: ['PLATFORM_MAC'], + platform: ['MAC'], }, expectedOverlapped: true, }, { filter1: { - platform: ['PLATFORM_WINDOWS'], + platform: ['WINDOWS'], }, filter2: { - platform: ['PLATFORM_MAC'], + platform: ['MAC'], }, expectedOverlapped: false, }, diff --git a/src/seed_tools/utils/study_json_utils.test.ts b/src/seed_tools/utils/study_json_utils.test.ts index 9c29d313..b645bce1 100644 --- a/src/seed_tools/utils/study_json_utils.test.ts +++ b/src/seed_tools/utils/study_json_utils.test.ts @@ -44,7 +44,7 @@ describe('stringifyStudies', () => { name: 'study', filter: { channel: ['CANARY', 'BETA', 'STABLE'], - platform: ['PLATFORM_LINUX', 'PLATFORM_MAC'], + platform: ['LINUX', 'MAC'], }, }, { ignoreUnknownFields: false }, @@ -112,7 +112,7 @@ describe('stringifyStudies', () => { filter: { start_date: Math.floor(startDate.getTime() / 1000), channel: ['CANARY', 'BETA', 'STABLE'], - platform: ['PLATFORM_LINUX', 'PLATFORM_MAC'], + platform: ['LINUX', 'MAC'], }, }, { ignoreUnknownFields: false }, @@ -127,7 +127,7 @@ describe('stringifyStudies', () => { filter: { start_date: startDate.toISOString(), channel: ['CANARY', 'BETA', 'STABLE'], - platform: ['PLATFORM_LINUX', 'PLATFORM_MAC'], + platform: ['LINUX', 'MAC'], }, }, ]); diff --git a/src/seed_tools/utils/study_validation.test.ts b/src/seed_tools/utils/study_validation.test.ts index 920eb2d9..8b75140a 100644 --- a/src/seed_tools/utils/study_validation.test.ts +++ b/src/seed_tools/utils/study_validation.test.ts @@ -30,7 +30,7 @@ describe('getStudyErrors', () => { filter: { locale: ['en'], channel: ['CANARY'], - platform: ['PLATFORM_WINDOWS'], + platform: ['WINDOWS'], start_date: Math.floor(new Date('2022-01-01').getTime() / 1000), end_date: Math.floor(new Date('2022-02-01').getTime() / 1000), min_version: '1.0', @@ -243,7 +243,7 @@ describe('getStudyErrors', () => { }, ], filter: { - platform: ['PLATFORM_LINUX'], + platform: ['LINUX'], channel: ['BETA'], }, }); @@ -266,7 +266,7 @@ describe('getStudyErrors', () => { }, ], filter: { - platform: ['PLATFORM_LINUX'], + platform: ['LINUX'], channel: ['BETA'], }, }); @@ -342,7 +342,7 @@ describe('getStudyErrors', () => { }, ], filter: { - platform: ['PLATFORM_LINUX'], + platform: ['LINUX'], channel: ['BETA'], }, }; @@ -523,7 +523,7 @@ describe('getStudyErrors', () => { }, ], filter: { - platform: ['PLATFORM_LINUX'], + platform: ['LINUX'], channel: ['BETA'], locale: [], exclude_locale: ['en'], @@ -614,7 +614,7 @@ describe('getStudyErrors', () => { probability_weight: 100, }, ], - filter: { ...filter, platform: ['PLATFORM_LINUX'], channel: ['BETA'] }, + filter: { ...filter, platform: ['LINUX'], channel: ['BETA'] }, }); expect(study_validation.getStudyErrors(study, studyFileBaseName)).toEqual( @@ -673,7 +673,7 @@ describe('getStudyErrors', () => { probability_weight: 100, }, ], - filter: { ...filter, platform: ['PLATFORM_LINUX'] }, + filter: { ...filter, platform: ['LINUX'] }, }); expect( @@ -685,7 +685,7 @@ describe('getStudyErrors', () => { test.each([ {}, { platform: [] }, - { platform: ['PLATFORM_LINUX', 'PLATFORM_LINUX'] }, + { platform: ['LINUX', 'LINUX'] }, ])('should error if platform is invalid', (filter: any) => { const study = Study.fromJson({ name: 'study', diff --git a/src/web/app/app.tsx b/src/web/app/app.tsx index b310b520..85fb84e1 100644 --- a/src/web/app/app.tsx +++ b/src/web/app/app.tsx @@ -12,7 +12,7 @@ import { priorityToText, type StudyFilter, } from '../../core/study_processor'; -import { variations as proto } from '../../proto/generated/proto_bundle'; +import { Study_CpuArchitecture, Study_FormFactor } from '../../proto/generated/study'; import { type ExperimentModel, type FeatureModel } from './experiment_model'; import { SearchParamManager } from './search_param_manager'; import { loadSeedDataAsync } from './seed_loader'; @@ -216,18 +216,18 @@ export function StudyItem(props: { study: StudyModel; filter: StudyFilter }) { /> proto.Study.FormFactor[e])} + include={filter?.form_factor?.map((e) => Study_FormFactor[e])} exclude={filter?.exclude_form_factor?.map( - (e) => proto.Study.FormFactor[e], + (e) => Study_FormFactor[e], )} /> proto.Study.CpuArchitecture[e], + (e) => Study_CpuArchitecture[e], )} exclude={filter?.exclude_cpu_architecture?.map( - (e) => proto.Study.CpuArchitecture[e], + (e) => Study_CpuArchitecture[e], )} /> {filter != null && diff --git a/src/web/app/experiment_model.ts b/src/web/app/experiment_model.ts index 1efacaef..882f8f6b 100644 --- a/src/web/app/experiment_model.ts +++ b/src/web/app/experiment_model.ts @@ -4,7 +4,7 @@ // You can obtain one at https://mozilla.org/MPL/2.0/. import * as url_utils from '../../core/url_utils'; -import { type variations as proto } from '../../proto/generated/proto_bundle'; +import { Study_Experiment } from '../../proto/generated/study'; import { type StudyModel } from './study_model'; export class FeatureModel { @@ -13,10 +13,10 @@ export class FeatureModel { } export class ExperimentModel { - private readonly experiment: proto.Study.IExperiment; + private readonly experiment: Study_Experiment; private readonly studyModel: StudyModel; - constructor(experiment: proto.Study.IExperiment, studyModel: StudyModel) { + constructor(experiment: Study_Experiment, studyModel: StudyModel) { this.experiment = experiment; this.studyModel = studyModel; } @@ -58,7 +58,7 @@ export class ExperimentModel { weight(): number { const totalWeight = this.studyModel.processedStudy.studyDetails.totalWeight; if (totalWeight === 0) return 0; - return (this.experiment.probability_weight / totalWeight) * 100; + return ((this.experiment.probability_weight ?? 0) / totalWeight) * 100; } isMajorGroup(): boolean { diff --git a/src/web/app/seed_loader.ts b/src/web/app/seed_loader.ts index e79deb1c..fb04f2c1 100644 --- a/src/web/app/seed_loader.ts +++ b/src/web/app/seed_loader.ts @@ -5,8 +5,8 @@ import { SeedType, type ProcessingOptions } from '../../core/base_types'; import { ProcessedStudy } from '../../core/study_processor'; -import { variations as proto } from '../../proto/generated/proto_bundle'; import { StudyListModel, StudyModel } from './study_model'; +import { VariationsSeed } from '../../proto/generated/variations_seed'; import * as url_utils from '../../core/url_utils'; @@ -47,7 +47,7 @@ async function loadFile( async function loadSeedFromUrl(url: string, type: SeedType) { const data = await loadFile(url, 'arraybuffer'); const seedBytes = new Uint8Array(data); - const seed = proto.VariationsSeed.decode(seedBytes); + const seed = VariationsSeed.fromBinary(seedBytes); const isBraveSeed = type !== SeedType.UPSTREAM; // Desktop/Android could use a different major chrome version. diff --git a/src/web/app/study_model.test.ts b/src/web/app/study_model.test.ts index 9431faa6..22553ef8 100644 --- a/src/web/app/study_model.test.ts +++ b/src/web/app/study_model.test.ts @@ -10,11 +10,12 @@ import { StudyFilter, StudyPriority, } from '../../core/study_processor'; -import { variations as proto } from '../../proto/generated/proto_bundle'; import { StudyListModel, StudyModel } from './study_model'; +import { Study, Study_Channel, Study_Platform } from '../../proto/generated/study'; +import { PartialMessage } from '@protobuf-ts/runtime'; -function makeStudyModel(properties: proto.IStudy) { - const study = new proto.Study(properties); +function makeStudyModel(properties: PartialMessage) { + const study = Study.create(properties); const processed = new ProcessedStudy(study, { minMajorVersion: 116, isBraveSeed: true, @@ -46,10 +47,10 @@ describe('models', () => { }, ], filter: { - channel: [proto.Study.Channel.STABLE, proto.Study.Channel.CANARY], + channel: [Study_Channel.STABLE, Study_Channel.CANARY], platform: [ - proto.Study.Platform.PLATFORM_WINDOWS, - proto.Study.Platform.PLATFORM_IOS, + Study_Platform.WINDOWS, + Study_Platform.IOS, ], }, }); @@ -60,11 +61,13 @@ describe('models', () => { { name: 'Some', probability_weight: 100, + param: [], + override_ui_string: [] }, ], filter: { - channel: [proto.Study.Channel.STABLE, proto.Study.Channel.CANARY], - platform: [proto.Study.Platform.PLATFORM_WINDOWS], + channel: [Study_Channel.STABLE, Study_Channel.CANARY], + platform: [Study_Platform.WINDOWS], max_version: '110.0.0.0', }, }); @@ -78,8 +81,8 @@ describe('models', () => { }, ], filter: { - channel: [proto.Study.Channel.BETA], - platform: [proto.Study.Platform.PLATFORM_WINDOWS], + channel: [Study_Channel.BETA], + platform: [Study_Platform.WINDOWS], }, }); diff --git a/src/web/app/study_model.ts b/src/web/app/study_model.ts index 1300d299..46b016e6 100644 --- a/src/web/app/study_model.ts +++ b/src/web/app/study_model.ts @@ -11,7 +11,7 @@ import { type StudyPriority, } from '../../core/study_processor'; import * as url_utils from '../../core/url_utils'; -import { type variations as proto } from '../../proto/generated/proto_bundle'; +import { Study_Filter } from '../../proto/generated/study'; import { ExperimentModel } from './experiment_model'; export class StudyModel { @@ -25,7 +25,7 @@ export class StudyModel { this.id = id; } - filter(): proto.Study.IFilter | undefined { + filter(): Study_Filter | undefined { return this.processedStudy.study.filter ?? undefined; } @@ -42,7 +42,11 @@ export class StudyModel { if (study.experiment == null) return []; const models: ExperimentModel[] = []; for (const e of study.experiment) { - if (e.probability_weight > 0 || f === undefined || f.showEmptyGroups) { + if ( + (e.probability_weight ?? 0) > 0 || + f === undefined || + f.showEmptyGroups + ) { const model = new ExperimentModel(e, this); models.push(model); } From 1d28b772106add837f2939fce4e8199c372170c1 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 6 Nov 2024 17:32:08 +0400 Subject: [PATCH 02/19] Fix lint --- src/core/serializers.ts | 7 +--- src/core/study_processor.ts | 11 +++++- src/core/summary.ts | 2 +- src/finch_tracker/main.ts | 2 +- src/finch_tracker/tracker_lib.test.ts | 2 +- src/seed_tools/utils/study_validation.test.ts | 37 +++++++++---------- src/web/app/app.tsx | 9 +++-- src/web/app/seed_loader.ts | 2 +- src/web/app/study_model.test.ts | 15 ++++---- 9 files changed, 45 insertions(+), 42 deletions(-) diff --git a/src/core/serializers.ts b/src/core/serializers.ts index 01cdd095..05310c19 100644 --- a/src/core/serializers.ts +++ b/src/core/serializers.ts @@ -2,7 +2,7 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -import {Study, Study_Channel, Study_Platform} from '../proto/generated/study'; +import { Study, Study_Channel, Study_Platform } from '../proto/generated/study'; export function getPlatformNameFromString(protoPlatfrom: string): string { const PREFIX = 'PLATFORM_'; @@ -30,10 +30,7 @@ export function getChannelName( protoChannel: Study_Channel, isBraveSpecific: boolean, ): string { - return getChannelNameFromString( - Study_Channel[protoChannel], - isBraveSpecific, - ); + return getChannelNameFromString(Study_Channel[protoChannel], isBraveSpecific); } function unixSecondToUTCString(unixTimeSeconds: number): string { diff --git a/src/core/study_processor.ts b/src/core/study_processor.ts index 36732fed..31c5f851 100644 --- a/src/core/study_processor.ts +++ b/src/core/study_processor.ts @@ -2,7 +2,13 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -import { Study, Study_Channel, Study_Experiment, Study_Filter, Study_Platform} from '../proto/generated/study'; +import { + Study, + Study_Channel, + Study_Experiment, + Study_Filter, + Study_Platform, +} from '../proto/generated/study'; import { type ProcessingOptions } from './base_types'; import { isFeatureBlocklisted, isStudyNameBlocklisted } from './blocklists'; import { matchesMaxVersion, parseVersionPattern } from './version'; @@ -227,7 +233,8 @@ export class StudyDetails { this.isBlocklisted ||= disabledFeatures?.some((n) => isFeatureBlocklisted(n)) ?? false; - this.isKillSwitch ||= (e.probability_weight ?? 0) > 0 && isKillSwitch(e.name); + this.isKillSwitch ||= + (e.probability_weight ?? 0) > 0 && isKillSwitch(e.name); this.onlyDisabledFeatures &&= e.probability_weight === 0 || diff --git a/src/core/summary.ts b/src/core/summary.ts index 117e2491..a2ee2acc 100644 --- a/src/core/summary.ts +++ b/src/core/summary.ts @@ -13,8 +13,8 @@ import { } from './study_processor'; import * as config from '../config'; -import * as url_utils from './url_utils'; import { VariationsSeed } from '../proto/generated/variations_seed'; +import * as url_utils from './url_utils'; export enum ItemAction { New, diff --git a/src/finch_tracker/main.ts b/src/finch_tracker/main.ts index 4c360881..27bd543c 100644 --- a/src/finch_tracker/main.ts +++ b/src/finch_tracker/main.ts @@ -8,13 +8,13 @@ import { type ProcessingOptions } from '../core/base_types'; import { StudyPriority } from '../core/study_processor'; import { makeSummary, summaryToJson } from '../core/summary'; import * as url_utils from '../core/url_utils'; +import { VariationsSeed } from '../proto/generated/variations_seed'; import { downloadUrl, getSeedPath } from './node_utils'; import { commitAllChanges, fetchChromeSeedData, storeDataToDirectory, } from './tracker_lib'; -import { VariationsSeed } from '../proto/generated/variations_seed'; async function main(): Promise { const program = new Command() diff --git a/src/finch_tracker/tracker_lib.test.ts b/src/finch_tracker/tracker_lib.test.ts index 4e7c8278..8f57c577 100644 --- a/src/finch_tracker/tracker_lib.test.ts +++ b/src/finch_tracker/tracker_lib.test.ts @@ -8,7 +8,7 @@ import * as fs from 'fs'; import { describe, expect, test } from '@jest/globals'; import { StudyPriority } from '../core/study_processor'; import { ItemAction, makeSummary, summaryToJson } from '../core/summary'; -import { Study, Study_Channel, Study_Platform} from '../proto/generated/study'; +import { Study, Study_Channel, Study_Platform } from '../proto/generated/study'; import { VariationsSeed } from '../proto/generated/variations_seed'; import { serializeStudies } from './tracker_lib'; diff --git a/src/seed_tools/utils/study_validation.test.ts b/src/seed_tools/utils/study_validation.test.ts index 8b75140a..5246d51c 100644 --- a/src/seed_tools/utils/study_validation.test.ts +++ b/src/seed_tools/utils/study_validation.test.ts @@ -682,24 +682,23 @@ describe('getStudyErrors', () => { }, ); - test.each([ - {}, - { platform: [] }, - { platform: ['LINUX', 'LINUX'] }, - ])('should error if platform is invalid', (filter: any) => { - const study = Study.fromJson({ - name: 'study', - experiment: [ - { - name: 'experiment1', - probability_weight: 100, - }, - ], - filter: { ...filter, channel: ['BETA'] }, - }); + test.each([{}, { platform: [] }, { platform: ['LINUX', 'LINUX'] }])( + 'should error if platform is invalid', + (filter: any) => { + const study = Study.fromJson({ + name: 'study', + experiment: [ + { + name: 'experiment1', + probability_weight: 100, + }, + ], + filter: { ...filter, channel: ['BETA'] }, + }); - expect( - study_validation.getStudyErrors(study, studyFileBaseName), - ).toContainEqual(expect.stringMatching(/(P|p)latform/)); - }); + expect( + study_validation.getStudyErrors(study, studyFileBaseName), + ).toContainEqual(expect.stringMatching(/(P|p)latform/)); + }, + ); }); diff --git a/src/web/app/app.tsx b/src/web/app/app.tsx index 85fb84e1..70af53ec 100644 --- a/src/web/app/app.tsx +++ b/src/web/app/app.tsx @@ -12,7 +12,10 @@ import { priorityToText, type StudyFilter, } from '../../core/study_processor'; -import { Study_CpuArchitecture, Study_FormFactor } from '../../proto/generated/study'; +import { + Study_CpuArchitecture, + Study_FormFactor, +} from '../../proto/generated/study'; import { type ExperimentModel, type FeatureModel } from './experiment_model'; import { SearchParamManager } from './search_param_manager'; import { loadSeedDataAsync } from './seed_loader'; @@ -217,9 +220,7 @@ export function StudyItem(props: { study: StudyModel; filter: StudyFilter }) { Study_FormFactor[e])} - exclude={filter?.exclude_form_factor?.map( - (e) => Study_FormFactor[e], - )} + exclude={filter?.exclude_form_factor?.map((e) => Study_FormFactor[e])} /> ) { const study = Study.create(properties); @@ -48,10 +52,7 @@ describe('models', () => { ], filter: { channel: [Study_Channel.STABLE, Study_Channel.CANARY], - platform: [ - Study_Platform.WINDOWS, - Study_Platform.IOS, - ], + platform: [Study_Platform.WINDOWS, Study_Platform.IOS], }, }); @@ -61,8 +62,6 @@ describe('models', () => { { name: 'Some', probability_weight: 100, - param: [], - override_ui_string: [] }, ], filter: { From 5451c979c231ec87021c682c44a719629da219eb Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 6 Nov 2024 17:36:06 +0400 Subject: [PATCH 03/19] Fix some tests --- src/core/serializers.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/serializers.ts b/src/core/serializers.ts index 05310c19..50f91001 100644 --- a/src/core/serializers.ts +++ b/src/core/serializers.ts @@ -50,7 +50,10 @@ export function serializeChannels(channels?: string[]): string | undefined { // Converts a study to JSON that is ready to be serialized. Some field are // removed, some are converted to a human readable format. export function studyToJSON(study: Study): Record { - const json = Study.toJson(study) as Record | null; + const json = Study.toJson(study, { useProtoFieldName: true }) as Record< + string, + any + > | null; if (json === null) { throw new Error('Failed to convert study to JSON'); } From 8a3fb45e29e0f269f5381b7f41f118f56535df0f Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 6 Nov 2024 17:41:02 +0400 Subject: [PATCH 04/19] Revert seed_tools changes --- src/seed_tools/utils/seed_validation.test.ts | 10 ++-- src/seed_tools/utils/study_json_utils.test.ts | 6 +-- src/seed_tools/utils/study_validation.test.ts | 51 ++++++++++--------- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/src/seed_tools/utils/seed_validation.test.ts b/src/seed_tools/utils/seed_validation.test.ts index ff215233..3a4ef2fb 100644 --- a/src/seed_tools/utils/seed_validation.test.ts +++ b/src/seed_tools/utils/seed_validation.test.ts @@ -31,7 +31,7 @@ describe('getSeedErrors', () => { } else { study.filter = {}; } - study.filter.platform = study.filter.platform ?? ['LINUX']; + study.filter.platform = study.filter.platform ?? ['PLATFORM_LINUX']; study.filter.channel = study.filter.channel ?? ['BETA']; return Study.fromJson(study, { ignoreUnknownFields: false, @@ -184,19 +184,19 @@ describe('getSeedErrors', () => { // platform tests { filter1: { - platform: ['WINDOWS', 'MAC'], + platform: ['PLATFORM_WINDOWS', 'PLATFORM_MAC'], }, filter2: { - platform: ['MAC'], + platform: ['PLATFORM_MAC'], }, expectedOverlapped: true, }, { filter1: { - platform: ['WINDOWS'], + platform: ['PLATFORM_WINDOWS'], }, filter2: { - platform: ['MAC'], + platform: ['PLATFORM_MAC'], }, expectedOverlapped: false, }, diff --git a/src/seed_tools/utils/study_json_utils.test.ts b/src/seed_tools/utils/study_json_utils.test.ts index b645bce1..9c29d313 100644 --- a/src/seed_tools/utils/study_json_utils.test.ts +++ b/src/seed_tools/utils/study_json_utils.test.ts @@ -44,7 +44,7 @@ describe('stringifyStudies', () => { name: 'study', filter: { channel: ['CANARY', 'BETA', 'STABLE'], - platform: ['LINUX', 'MAC'], + platform: ['PLATFORM_LINUX', 'PLATFORM_MAC'], }, }, { ignoreUnknownFields: false }, @@ -112,7 +112,7 @@ describe('stringifyStudies', () => { filter: { start_date: Math.floor(startDate.getTime() / 1000), channel: ['CANARY', 'BETA', 'STABLE'], - platform: ['LINUX', 'MAC'], + platform: ['PLATFORM_LINUX', 'PLATFORM_MAC'], }, }, { ignoreUnknownFields: false }, @@ -127,7 +127,7 @@ describe('stringifyStudies', () => { filter: { start_date: startDate.toISOString(), channel: ['CANARY', 'BETA', 'STABLE'], - platform: ['LINUX', 'MAC'], + platform: ['PLATFORM_LINUX', 'PLATFORM_MAC'], }, }, ]); diff --git a/src/seed_tools/utils/study_validation.test.ts b/src/seed_tools/utils/study_validation.test.ts index 5246d51c..920eb2d9 100644 --- a/src/seed_tools/utils/study_validation.test.ts +++ b/src/seed_tools/utils/study_validation.test.ts @@ -30,7 +30,7 @@ describe('getStudyErrors', () => { filter: { locale: ['en'], channel: ['CANARY'], - platform: ['WINDOWS'], + platform: ['PLATFORM_WINDOWS'], start_date: Math.floor(new Date('2022-01-01').getTime() / 1000), end_date: Math.floor(new Date('2022-02-01').getTime() / 1000), min_version: '1.0', @@ -243,7 +243,7 @@ describe('getStudyErrors', () => { }, ], filter: { - platform: ['LINUX'], + platform: ['PLATFORM_LINUX'], channel: ['BETA'], }, }); @@ -266,7 +266,7 @@ describe('getStudyErrors', () => { }, ], filter: { - platform: ['LINUX'], + platform: ['PLATFORM_LINUX'], channel: ['BETA'], }, }); @@ -342,7 +342,7 @@ describe('getStudyErrors', () => { }, ], filter: { - platform: ['LINUX'], + platform: ['PLATFORM_LINUX'], channel: ['BETA'], }, }; @@ -523,7 +523,7 @@ describe('getStudyErrors', () => { }, ], filter: { - platform: ['LINUX'], + platform: ['PLATFORM_LINUX'], channel: ['BETA'], locale: [], exclude_locale: ['en'], @@ -614,7 +614,7 @@ describe('getStudyErrors', () => { probability_weight: 100, }, ], - filter: { ...filter, platform: ['LINUX'], channel: ['BETA'] }, + filter: { ...filter, platform: ['PLATFORM_LINUX'], channel: ['BETA'] }, }); expect(study_validation.getStudyErrors(study, studyFileBaseName)).toEqual( @@ -673,7 +673,7 @@ describe('getStudyErrors', () => { probability_weight: 100, }, ], - filter: { ...filter, platform: ['LINUX'] }, + filter: { ...filter, platform: ['PLATFORM_LINUX'] }, }); expect( @@ -682,23 +682,24 @@ describe('getStudyErrors', () => { }, ); - test.each([{}, { platform: [] }, { platform: ['LINUX', 'LINUX'] }])( - 'should error if platform is invalid', - (filter: any) => { - const study = Study.fromJson({ - name: 'study', - experiment: [ - { - name: 'experiment1', - probability_weight: 100, - }, - ], - filter: { ...filter, channel: ['BETA'] }, - }); + test.each([ + {}, + { platform: [] }, + { platform: ['PLATFORM_LINUX', 'PLATFORM_LINUX'] }, + ])('should error if platform is invalid', (filter: any) => { + const study = Study.fromJson({ + name: 'study', + experiment: [ + { + name: 'experiment1', + probability_weight: 100, + }, + ], + filter: { ...filter, channel: ['BETA'] }, + }); - expect( - study_validation.getStudyErrors(study, studyFileBaseName), - ).toContainEqual(expect.stringMatching(/(P|p)latform/)); - }, - ); + expect( + study_validation.getStudyErrors(study, studyFileBaseName), + ).toContainEqual(expect.stringMatching(/(P|p)latform/)); + }); }); From b36b5937ce0249530c200f78a0b2fe2efd951662 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 6 Nov 2024 17:42:56 +0400 Subject: [PATCH 05/19] Fix tests --- src/test/data/seed1.bin.processing_expectations | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/data/seed1.bin.processing_expectations b/src/test/data/seed1.bin.processing_expectations index 46a591ec..4a861f33 100644 --- a/src/test/data/seed1.bin.processing_expectations +++ b/src/test/data/seed1.bin.processing_expectations @@ -54,9 +54,9 @@ } ], "filter": { + "end_date": "Wed, 14 Sep 2022 17:10:24 GMT", "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID", - "end_date": "Wed, 14 Sep 2022 17:10:24 GMT" + "platform": "WINDOWS, MAC, LINUX, ANDROID" } } ], From 29c8674dde895b9980e13e9ce63b4c30ea4ebd4f Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 6 Nov 2024 17:45:05 +0400 Subject: [PATCH 06/19] Remove protobufjs --- package.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/package.json b/package.json index 75f35f5d..f3e84c11 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,6 @@ "prettier": "3.3.3", "prettier-plugin-multiline-arrays": "3.0.6", "prettier-plugin-organize-imports": "3.2.4", - "protobufjs": "7.4.0", - "protobufjs-cli": "1.1.3", "style-loader": "3.3.4", "ts-jest": "29.2.5", "ts-loader": "9.5.1", From 5b5716cc62f08320a6c8e0dcd3f548d9747f02f3 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 6 Nov 2024 17:49:10 +0400 Subject: [PATCH 07/19] Fix proto gen --- src/scripts/generate_proto_ts.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scripts/generate_proto_ts.ts b/src/scripts/generate_proto_ts.ts index 9d8408ad..de01ba7f 100644 --- a/src/scripts/generate_proto_ts.ts +++ b/src/scripts/generate_proto_ts.ts @@ -74,7 +74,7 @@ function generateProtobufTs() { function generateStudyProtoPatch() { fs.writeFileSync( - `${protoDir}/study.protobuf-ts.patch`, + `${protoDir}/study.proto.protobuf-ts.patch`, execSync(`git diff ${protoDir}/study.proto`, { encoding: 'buffer', }), @@ -82,14 +82,14 @@ function generateStudyProtoPatch() { } function gitApplyStudyProtoPatch() { - execSync(`git apply ${protoDir}/study.protobuf-ts.patch`, { + execSync(`git apply ${protoDir}/study.proto.protobuf-ts.patch`, { cwd: protoDir, stdio: 'inherit', }); } function gitRevertStudyProtoPatch() { - execSync(`git apply -R ${protoDir}/study.protobuf-ts.patch`, { + execSync(`git apply -R ${protoDir}/study.proto.protobuf-ts.patch`, { cwd: protoDir, stdio: 'inherit', }); From 1619c74316fc2e57384b273faaae419729061da2 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 6 Nov 2024 18:49:23 +0400 Subject: [PATCH 08/19] Use seed_tool serialization --- src/core/serializers.ts | 41 +- src/finch_tracker/main.ts | 2 +- src/finch_tracker/tracker_lib.test.ts | 36 +- src/finch_tracker/tracker_lib.ts | 27 +- .../data/seed1.bin.processing_expectations | 694 +++++++++++------- 5 files changed, 455 insertions(+), 345 deletions(-) diff --git a/src/core/serializers.ts b/src/core/serializers.ts index 50f91001..4da2cb09 100644 --- a/src/core/serializers.ts +++ b/src/core/serializers.ts @@ -2,7 +2,7 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -import { Study, Study_Channel, Study_Platform } from '../proto/generated/study'; +import { Study_Channel, Study_Platform } from '../proto/generated/study'; export function getPlatformNameFromString(protoPlatfrom: string): string { const PREFIX = 'PLATFORM_'; @@ -32,42 +32,3 @@ export function getChannelName( ): string { return getChannelNameFromString(Study_Channel[protoChannel], isBraveSpecific); } - -function unixSecondToUTCString(unixTimeSeconds: number): string { - return new Date(unixTimeSeconds * 1000).toUTCString(); -} - -export function serializePlatforms(platforms?: string[]): string | undefined { - if (platforms === undefined) return undefined; - return platforms.map((v) => getPlatformNameFromString(v)).join(', '); -} - -export function serializeChannels(channels?: string[]): string | undefined { - if (channels === undefined) return undefined; - return channels.join(', '); -} - -// Converts a study to JSON that is ready to be serialized. Some field are -// removed, some are converted to a human readable format. -export function studyToJSON(study: Study): Record { - const json = Study.toJson(study, { useProtoFieldName: true }) as Record< - string, - any - > | null; - if (json === null) { - throw new Error('Failed to convert study to JSON'); - } - const filter = json.filter; - delete json.consistency; - delete json.activation_type; - if (filter !== undefined) { - if (filter.end_date !== undefined) - filter.end_date = unixSecondToUTCString(filter.end_date); - if (filter.start_date !== undefined) { - filter.start_date = unixSecondToUTCString(filter.start_date); - } - filter.platform = serializePlatforms(filter.platform); - filter.channel = serializeChannels(filter.channel); - } - return json; -} diff --git a/src/finch_tracker/main.ts b/src/finch_tracker/main.ts index 27bd543c..b3b9647b 100644 --- a/src/finch_tracker/main.ts +++ b/src/finch_tracker/main.ts @@ -98,7 +98,7 @@ async function main(): Promise { } if (updateData) { - storeDataToDirectory(seedData, storageDir, options); + await storeDataToDirectory(seedData, storageDir, options); if (commitData) { newGitSha1 = commitAllChanges(storageDir); } diff --git a/src/finch_tracker/tracker_lib.test.ts b/src/finch_tracker/tracker_lib.test.ts index 8f57c577..7967fdb6 100644 --- a/src/finch_tracker/tracker_lib.test.ts +++ b/src/finch_tracker/tracker_lib.test.ts @@ -4,32 +4,42 @@ // You can obtain one at https://mozilla.org/MPL/2.0/. import * as fs from 'fs'; +import * as os from 'os'; import { describe, expect, test } from '@jest/globals'; +import path from 'path'; import { StudyPriority } from '../core/study_processor'; import { ItemAction, makeSummary, summaryToJson } from '../core/summary'; import { Study, Study_Channel, Study_Platform } from '../proto/generated/study'; import { VariationsSeed } from '../proto/generated/variations_seed'; -import { serializeStudies } from './tracker_lib'; - -function serialize(json: Record) { - const ordered = Object.keys(json) - .sort() - .reduce((res: Record, key) => { - res[key] = json[key]; - return res; - }, {}); - return JSON.stringify(ordered, undefined, 2); +import { storeDataToDirectory } from './tracker_lib'; + +function readDirectory(dir: string): string { + const files = fs + .readdirSync(dir, { recursive: true, encoding: 'utf-8' }) + .sort(); + let result = ''; + + for (const file of files) { + const filePath = path.join(dir, file); + if (!file.endsWith('.json5')) { + continue; + } + const content = fs.readFileSync(filePath, 'utf-8'); + result += file + '\n' + content + '\n'; + } + return result; } -test('seed serialization', () => { +test('seed serialization', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'tracker-')); const data = fs.readFileSync('src/test/data/seed1.bin'); - const map = serializeStudies(data, { + await storeDataToDirectory(data, tempDir, { minMajorVersion: 116, isBraveSeed: true, }); - const serializedOutput = serialize(map); + const serializedOutput = readDirectory(path.join(tempDir)); const serializedExpectations = fs .readFileSync('src/test/data/seed1.bin.processing_expectations') .toString(); diff --git a/src/finch_tracker/tracker_lib.ts b/src/finch_tracker/tracker_lib.ts index 7c1149a6..6a752ebb 100644 --- a/src/finch_tracker/tracker_lib.ts +++ b/src/finch_tracker/tracker_lib.ts @@ -8,7 +8,6 @@ import * as fs from 'fs'; import * as path from 'path'; import { type ProcessingOptions } from '../core/base_types'; -import { studyToJSON } from '../core/serializers'; import { ProcessedStudy, StudyPriority, @@ -16,6 +15,7 @@ import { } from '../core/study_processor'; import { Study } from '../proto/generated/study'; import { VariationsSeed } from '../proto/generated/variations_seed'; +import { writeStudyFile } from '../seed_tools/utils/study_json_utils'; import { downloadUrl, getSeedPath, getStudyPath } from './node_utils'; export async function fetchChromeSeedData(): Promise { @@ -24,18 +24,17 @@ export async function fetchChromeSeedData(): Promise { return await downloadUrl(kChromeSeedUrl); } -// Processes, groups by name and converts to JSON a list of studies. -export function serializeStudies( +// Groups studies by name and priority. +export function groupStudies( seedData: Buffer, options: ProcessingOptions, -): Record { - const map: Record = {}; +): Record { + const map: Record = {}; const seed = VariationsSeed.fromBinary(seedData); const addStudy = (path: string, study: Study) => { - const json = studyToJSON(study); const list = map[path]; - if (list !== undefined) list.push(json); - else map[path] = [json]; + if (list !== undefined) list.push(study); + else map[path] = [study]; }; for (const study of seed.study) { @@ -74,20 +73,20 @@ export function commitAllChanges(directory: string): string | undefined { // Processes and serializes a given seed to disk (including grouping to // subdirectories/files). -export function storeDataToDirectory( +export async function storeDataToDirectory( seedData: Buffer, directory: string, options: ProcessingOptions, -): void { +): Promise { const studyDirectory = getStudyPath(directory); fs.rmSync(studyDirectory, { recursive: true, force: true }); - const map = serializeStudies(seedData, options); + const map = groupStudies(seedData, options); - for (const [name, json] of Object.entries(map)) { - const fileName = `${studyDirectory}/${name}`; + for (const [name, study] of Object.entries(map)) { + const fileName = `${studyDirectory}/${name}.json5`; const dirname = path.dirname(fileName); fs.mkdirSync(dirname, { recursive: true }); - fs.writeFileSync(fileName, JSON.stringify(json, null, 2) + '\n'); + await writeStudyFile(study, fileName, { isChromium: !options.isBraveSeed }); } // TODO: maybe start to use s3 instead of git one day? diff --git a/src/test/data/seed1.bin.processing_expectations b/src/test/data/seed1.bin.processing_expectations index 4a861f33..136a186b 100644 --- a/src/test/data/seed1.bin.processing_expectations +++ b/src/test/data/seed1.bin.processing_expectations @@ -1,298 +1,438 @@ -{ - "all-by-name/BetaStudy": [ - { - "name": "BetaStudy", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } - } +study/all-by-name/BetaStudy.json5 +[ + { + name: 'BetaStudy', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, + }, + ], + filter: { + channel: [ + 'BETA', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', ], - "filter": { - "channel": "BETA", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/BlocklistedStudy": [ - { - "name": "BlocklistedStudy", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "Ukm" - ] - } - } + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/BlocklistedStudy.json5 +[ + { + name: 'BlocklistedStudy', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'Ukm', + ], + }, + }, + ], + filter: { + channel: [ + 'RELEASE', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/EndedByDate": [ - { - "name": "EndedByDate", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/EndedByDate.json5 +[ + { + name: 'EndedByDate', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, + }, + ], + filter: { + end_date: '2022-09-14T17:10:24.000Z', + channel: [ + 'RELEASE', ], - "filter": { - "end_date": "Wed, 14 Sep 2022 17:10:24 GMT", - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/EndedMaxVersion": [ - { - "name": "EndedMaxVersion", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', ], - "filter": { - "max_version": "99.1.49.83", - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/Stable-100": [ - { - "name": "Stable-100", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 99, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/EndedMaxVersion.json5 +[ + { + name: 'EndedMaxVersion', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Default", - "probability_weight": 1 - } + }, + ], + filter: { + max_version: '99.1.49.83', + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/Stable-100.json5 +[ + { + name: 'Stable-100', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 99, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, + }, + { + name: 'Default', + probability_weight: 1, + }, + ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/Stable-50": [ - { - "name": "Stable-50", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 50, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/Stable-50.json5 +[ + { + name: 'Stable-50', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 50, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Default", - "probability_weight": 50 - } + }, + { + name: 'Default', + probability_weight: 50, + }, + ], + filter: { + channel: [ + 'RELEASE', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/Stable-min": [ - { - "name": "Stable-min", - "experiment": [ - { - "name": "Enabled_1", - "probability_weight": 5, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/Stable-min.json5 +[ + { + name: 'Stable-min', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled_1', + probability_weight: 5, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Control_1", - "probability_weight": 45 + }, + { + name: 'Control_1', + probability_weight: 45, + }, + { + name: 'Default', + probability_weight: 50, + }, + ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/all-by-name/StudyKillSwitch.json5 +[ + { + name: 'StudyKillSwitch', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Default", - "probability_weight": 50 - } + }, + ], + filter: { + max_version: '299.0.0.0', + channel: [ + 'RELEASE', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "all-by-name/StudyKillSwitch": [ - { - "name": "StudyKillSwitch", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/blocklisted/BlocklistedStudy.json5 +[ + { + name: 'BlocklistedStudy', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'Ukm', + ], + }, + }, + ], + filter: { + channel: [ + 'RELEASE', ], - "filter": { - "max_version": "299.0.0.0", - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "blocklisted/BlocklistedStudy": [ - { - "name": "BlocklistedStudy", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "Ukm" - ] - } - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "stable-100%/Stable-100": [ - { - "name": "Stable-100", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 99, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/stable-100%/Stable-100.json5 +[ + { + name: 'Stable-100', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 99, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Default", - "probability_weight": 1 - } + }, + { + name: 'Default', + probability_weight: 1, + }, + ], + filter: { + channel: [ + 'RELEASE', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "stable-50%/Stable-50": [ - { - "name": "Stable-50", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 50, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/stable-50%/Stable-50.json5 +[ + { + name: 'Stable-50', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 50, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Default", - "probability_weight": 50 - } + }, + { + name: 'Default', + probability_weight: 50, + }, + ], + filter: { + channel: [ + 'RELEASE', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "stable-emergency-kill-switch/StudyKillSwitch": [ - { - "name": "StudyKillSwitch", - "experiment": [ - { - "name": "Enabled", - "probability_weight": 100, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } - } + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', ], - "filter": { - "max_version": "299.0.0.0", - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ], - "stable-min/Stable-min": [ - { - "name": "Stable-min", - "experiment": [ - { - "name": "Enabled_1", - "probability_weight": 5, - "feature_association": { - "enable_feature": [ - "SomeFeature" - ] - } + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/stable-emergency-kill-switch/StudyKillSwitch.json5 +[ + { + name: 'StudyKillSwitch', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Control_1", - "probability_weight": 45 + }, + ], + filter: { + max_version: '299.0.0.0', + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] + +study/stable-min/Stable-min.json5 +[ + { + name: 'Stable-min', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled_1', + probability_weight: 5, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], }, - { - "name": "Default", - "probability_weight": 50 - } + }, + { + name: 'Control_1', + probability_weight: 45, + }, + { + name: 'Default', + probability_weight: 50, + }, + ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', ], - "filter": { - "channel": "STABLE", - "platform": "WINDOWS, MAC, LINUX, ANDROID" - } - } - ] -} \ No newline at end of file + }, + activation_type: 'ACTIVATE_ON_STARTUP', + }, +] From 036363234f373410e79db15f3d49f1fefd2aa28a Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 6 Nov 2024 18:55:19 +0400 Subject: [PATCH 09/19] Adjust URLs --- src/core/summary.ts | 2 +- src/core/url_utils.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/summary.ts b/src/core/summary.ts index a2ee2acc..82266ff7 100644 --- a/src/core/summary.ts +++ b/src/core/summary.ts @@ -215,7 +215,7 @@ function getGitHubDiffUrl( oldPriority: StudyPriority, commit: string, ): string { - const path = `study/all-by-name/${study}`; + const path = `study/all-by-name/${study}.json5`; const pathHash = sha256(path); return `${url_utils.getGitHubStorageUrl()}/commit/${commit}#diff-${pathHash}`; } diff --git a/src/core/url_utils.ts b/src/core/url_utils.ts index 101c1775..4f838a84 100644 --- a/src/core/url_utils.ts +++ b/src/core/url_utils.ts @@ -51,8 +51,7 @@ export function getStudyRawConfigUrl( return ( 'https://github.com/search?type=code' + '&q=repo%3Abrave%2Fbrave-variations' + - '+path%3A%2F%5Eseed%5C%2Fseed.json%7C%5Estudies%5C%2F*.json5%2F+' + - `"%5C"name%5C"%3A+%5C"${encodeURIComponent(study)}%5C""` + `+path%3Astudies%2F+%22name%3A+%27${encodeURIComponent(study)}%27%22` ); } From 37cd0d199e3a1167f90ecd59b2fc68309590fe13 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 6 Nov 2024 19:04:48 +0400 Subject: [PATCH 10/19] Fix tests --- src/finch_tracker/tracker_lib.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/finch_tracker/tracker_lib.test.ts b/src/finch_tracker/tracker_lib.test.ts index 7967fdb6..9b7eeb9a 100644 --- a/src/finch_tracker/tracker_lib.test.ts +++ b/src/finch_tracker/tracker_lib.test.ts @@ -26,7 +26,8 @@ function readDirectory(dir: string): string { continue; } const content = fs.readFileSync(filePath, 'utf-8'); - result += file + '\n' + content + '\n'; + if (result != '') result += '\n'; + result += file + '\n' + content; } return result; } From 0886bb4e3af976c6ee8b914a548f1911deece510 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 20 Nov 2024 14:24:07 +0700 Subject: [PATCH 11/19] update seed1.bin.processing_expectations format --- src/finch_tracker/tracker_lib.test.ts | 13 +- .../data/seed1.bin.processing_expectations | 775 +++++++++--------- 2 files changed, 384 insertions(+), 404 deletions(-) diff --git a/src/finch_tracker/tracker_lib.test.ts b/src/finch_tracker/tracker_lib.test.ts index 9b7eeb9a..60217c5e 100644 --- a/src/finch_tracker/tracker_lib.test.ts +++ b/src/finch_tracker/tracker_lib.test.ts @@ -13,12 +13,13 @@ import { ItemAction, makeSummary, summaryToJson } from '../core/summary'; import { Study, Study_Channel, Study_Platform } from '../proto/generated/study'; import { VariationsSeed } from '../proto/generated/variations_seed'; import { storeDataToDirectory } from './tracker_lib'; +import JSON5 from 'json5'; -function readDirectory(dir: string): string { +function readDirectory(dir: string): Record { const files = fs .readdirSync(dir, { recursive: true, encoding: 'utf-8' }) .sort(); - let result = ''; + const result: Record = {}; for (const file of files) { const filePath = path.join(dir, file); @@ -26,8 +27,7 @@ function readDirectory(dir: string): string { continue; } const content = fs.readFileSync(filePath, 'utf-8'); - if (result != '') result += '\n'; - result += file + '\n' + content; + result[file] = JSON5.parse(content); } return result; } @@ -40,7 +40,10 @@ test('seed serialization', async () => { isBraveSeed: true, }); - const serializedOutput = readDirectory(path.join(tempDir)); + const serializedOutput = JSON5.stringify( + readDirectory(path.join(tempDir)), + { space: 2 }, + ); const serializedExpectations = fs .readFileSync('src/test/data/seed1.bin.processing_expectations') .toString(); diff --git a/src/test/data/seed1.bin.processing_expectations b/src/test/data/seed1.bin.processing_expectations index 136a186b..b6d42d81 100644 --- a/src/test/data/seed1.bin.processing_expectations +++ b/src/test/data/seed1.bin.processing_expectations @@ -1,438 +1,415 @@ -study/all-by-name/BetaStudy.json5 -[ - { - name: 'BetaStudy', - consistency: 'PERMANENT', - experiment: [ - { - name: 'Enabled', - probability_weight: 100, - feature_association: { - enable_feature: [ - 'SomeFeature', - ], +{ + 'study/all-by-name/BetaStudy.json5': [ + { + name: 'BetaStudy', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, }, - }, - ], - filter: { - channel: [ - 'BETA', - ], - platform: [ - 'WINDOWS', - 'MAC', - 'LINUX', - 'ANDROID', ], + filter: { + channel: [ + 'BETA', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', }, - activation_type: 'ACTIVATE_ON_STARTUP', - }, -] - -study/all-by-name/BlocklistedStudy.json5 -[ - { - name: 'BlocklistedStudy', - consistency: 'PERMANENT', - experiment: [ - { - name: 'Enabled', - probability_weight: 100, - feature_association: { - enable_feature: [ - 'Ukm', - ], + ], + 'study/all-by-name/BlocklistedStudy.json5': [ + { + name: 'BlocklistedStudy', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'Ukm', + ], + }, }, - }, - ], - filter: { - channel: [ - 'RELEASE', - ], - platform: [ - 'WINDOWS', - 'MAC', - 'LINUX', - 'ANDROID', ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', }, - activation_type: 'ACTIVATE_ON_STARTUP', - }, -] - -study/all-by-name/EndedByDate.json5 -[ - { - name: 'EndedByDate', - consistency: 'PERMANENT', - experiment: [ - { - name: 'Enabled', - probability_weight: 100, - feature_association: { - enable_feature: [ - 'SomeFeature', - ], + ], + 'study/all-by-name/EndedByDate.json5': [ + { + name: 'EndedByDate', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, }, - }, - ], - filter: { - end_date: '2022-09-14T17:10:24.000Z', - channel: [ - 'RELEASE', - ], - platform: [ - 'WINDOWS', - 'MAC', - 'LINUX', - 'ANDROID', ], + filter: { + end_date: '2022-09-14T17:10:24.000Z', + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', }, - activation_type: 'ACTIVATE_ON_STARTUP', - }, -] - -study/all-by-name/EndedMaxVersion.json5 -[ - { - name: 'EndedMaxVersion', - consistency: 'PERMANENT', - experiment: [ - { - name: 'Enabled', - probability_weight: 100, - feature_association: { - enable_feature: [ - 'SomeFeature', - ], + ], + 'study/all-by-name/EndedMaxVersion.json5': [ + { + name: 'EndedMaxVersion', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, }, - }, - ], - filter: { - max_version: '99.1.49.83', - channel: [ - 'RELEASE', - ], - platform: [ - 'WINDOWS', - 'MAC', - 'LINUX', - 'ANDROID', ], + filter: { + max_version: '99.1.49.83', + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', }, - activation_type: 'ACTIVATE_ON_STARTUP', - }, -] - -study/all-by-name/Stable-100.json5 -[ - { - name: 'Stable-100', - consistency: 'PERMANENT', - experiment: [ - { - name: 'Enabled', - probability_weight: 99, - feature_association: { - enable_feature: [ - 'SomeFeature', - ], + ], + 'study/all-by-name/Stable-100.json5': [ + { + name: 'Stable-100', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 99, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, + }, + { + name: 'Default', + probability_weight: 1, }, - }, - { - name: 'Default', - probability_weight: 1, - }, - ], - filter: { - channel: [ - 'RELEASE', - ], - platform: [ - 'WINDOWS', - 'MAC', - 'LINUX', - 'ANDROID', ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', }, - activation_type: 'ACTIVATE_ON_STARTUP', - }, -] - -study/all-by-name/Stable-50.json5 -[ - { - name: 'Stable-50', - consistency: 'PERMANENT', - experiment: [ - { - name: 'Enabled', - probability_weight: 50, - feature_association: { - enable_feature: [ - 'SomeFeature', - ], + ], + 'study/all-by-name/Stable-50.json5': [ + { + name: 'Stable-50', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 50, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, + }, + { + name: 'Default', + probability_weight: 50, }, - }, - { - name: 'Default', - probability_weight: 50, - }, - ], - filter: { - channel: [ - 'RELEASE', - ], - platform: [ - 'WINDOWS', - 'MAC', - 'LINUX', - 'ANDROID', ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', }, - activation_type: 'ACTIVATE_ON_STARTUP', - }, -] - -study/all-by-name/Stable-min.json5 -[ - { - name: 'Stable-min', - consistency: 'PERMANENT', - experiment: [ - { - name: 'Enabled_1', - probability_weight: 5, - feature_association: { - enable_feature: [ - 'SomeFeature', - ], + ], + 'study/all-by-name/Stable-min.json5': [ + { + name: 'Stable-min', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled_1', + probability_weight: 5, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, + }, + { + name: 'Control_1', + probability_weight: 45, + }, + { + name: 'Default', + probability_weight: 50, }, - }, - { - name: 'Control_1', - probability_weight: 45, - }, - { - name: 'Default', - probability_weight: 50, - }, - ], - filter: { - channel: [ - 'RELEASE', - ], - platform: [ - 'WINDOWS', - 'MAC', - 'LINUX', - 'ANDROID', ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', }, - activation_type: 'ACTIVATE_ON_STARTUP', - }, -] - -study/all-by-name/StudyKillSwitch.json5 -[ - { - name: 'StudyKillSwitch', - consistency: 'PERMANENT', - experiment: [ - { - name: 'Enabled', - probability_weight: 100, - feature_association: { - enable_feature: [ - 'SomeFeature', - ], + ], + 'study/all-by-name/StudyKillSwitch.json5': [ + { + name: 'StudyKillSwitch', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, }, - }, - ], - filter: { - max_version: '299.0.0.0', - channel: [ - 'RELEASE', - ], - platform: [ - 'WINDOWS', - 'MAC', - 'LINUX', - 'ANDROID', ], + filter: { + max_version: '299.0.0.0', + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', }, - activation_type: 'ACTIVATE_ON_STARTUP', - }, -] - -study/blocklisted/BlocklistedStudy.json5 -[ - { - name: 'BlocklistedStudy', - consistency: 'PERMANENT', - experiment: [ - { - name: 'Enabled', - probability_weight: 100, - feature_association: { - enable_feature: [ - 'Ukm', - ], + ], + 'study/blocklisted/BlocklistedStudy.json5': [ + { + name: 'BlocklistedStudy', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'Ukm', + ], + }, }, - }, - ], - filter: { - channel: [ - 'RELEASE', - ], - platform: [ - 'WINDOWS', - 'MAC', - 'LINUX', - 'ANDROID', ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', }, - activation_type: 'ACTIVATE_ON_STARTUP', - }, -] - -study/stable-100%/Stable-100.json5 -[ - { - name: 'Stable-100', - consistency: 'PERMANENT', - experiment: [ - { - name: 'Enabled', - probability_weight: 99, - feature_association: { - enable_feature: [ - 'SomeFeature', - ], + ], + 'study/stable-100%/Stable-100.json5': [ + { + name: 'Stable-100', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 99, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, + }, + { + name: 'Default', + probability_weight: 1, }, - }, - { - name: 'Default', - probability_weight: 1, - }, - ], - filter: { - channel: [ - 'RELEASE', - ], - platform: [ - 'WINDOWS', - 'MAC', - 'LINUX', - 'ANDROID', ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', }, - activation_type: 'ACTIVATE_ON_STARTUP', - }, -] - -study/stable-50%/Stable-50.json5 -[ - { - name: 'Stable-50', - consistency: 'PERMANENT', - experiment: [ - { - name: 'Enabled', - probability_weight: 50, - feature_association: { - enable_feature: [ - 'SomeFeature', - ], + ], + 'study/stable-50%/Stable-50.json5': [ + { + name: 'Stable-50', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 50, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, + }, + { + name: 'Default', + probability_weight: 50, }, - }, - { - name: 'Default', - probability_weight: 50, - }, - ], - filter: { - channel: [ - 'RELEASE', - ], - platform: [ - 'WINDOWS', - 'MAC', - 'LINUX', - 'ANDROID', ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', }, - activation_type: 'ACTIVATE_ON_STARTUP', - }, -] - -study/stable-emergency-kill-switch/StudyKillSwitch.json5 -[ - { - name: 'StudyKillSwitch', - consistency: 'PERMANENT', - experiment: [ - { - name: 'Enabled', - probability_weight: 100, - feature_association: { - enable_feature: [ - 'SomeFeature', - ], + ], + 'study/stable-emergency-kill-switch/StudyKillSwitch.json5': [ + { + name: 'StudyKillSwitch', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled', + probability_weight: 100, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, }, - }, - ], - filter: { - max_version: '299.0.0.0', - channel: [ - 'RELEASE', - ], - platform: [ - 'WINDOWS', - 'MAC', - 'LINUX', - 'ANDROID', ], + filter: { + max_version: '299.0.0.0', + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', }, - activation_type: 'ACTIVATE_ON_STARTUP', - }, -] - -study/stable-min/Stable-min.json5 -[ - { - name: 'Stable-min', - consistency: 'PERMANENT', - experiment: [ - { - name: 'Enabled_1', - probability_weight: 5, - feature_association: { - enable_feature: [ - 'SomeFeature', - ], + ], + 'study/stable-min/Stable-min.json5': [ + { + name: 'Stable-min', + consistency: 'PERMANENT', + experiment: [ + { + name: 'Enabled_1', + probability_weight: 5, + feature_association: { + enable_feature: [ + 'SomeFeature', + ], + }, + }, + { + name: 'Control_1', + probability_weight: 45, + }, + { + name: 'Default', + probability_weight: 50, }, - }, - { - name: 'Control_1', - probability_weight: 45, - }, - { - name: 'Default', - probability_weight: 50, - }, - ], - filter: { - channel: [ - 'RELEASE', - ], - platform: [ - 'WINDOWS', - 'MAC', - 'LINUX', - 'ANDROID', ], + filter: { + channel: [ + 'RELEASE', + ], + platform: [ + 'WINDOWS', + 'MAC', + 'LINUX', + 'ANDROID', + ], + }, + activation_type: 'ACTIVATE_ON_STARTUP', }, - activation_type: 'ACTIVATE_ON_STARTUP', - }, -] + ], +} \ No newline at end of file From ac905f2a10564b153e49fbba20934c8596f15202 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 20 Nov 2024 16:20:38 +0700 Subject: [PATCH 12/19] fix lint --- src/finch_tracker/tracker_lib.test.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/finch_tracker/tracker_lib.test.ts b/src/finch_tracker/tracker_lib.test.ts index 60217c5e..25447d5f 100644 --- a/src/finch_tracker/tracker_lib.test.ts +++ b/src/finch_tracker/tracker_lib.test.ts @@ -7,13 +7,13 @@ import * as fs from 'fs'; import * as os from 'os'; import { describe, expect, test } from '@jest/globals'; +import JSON5 from 'json5'; import path from 'path'; import { StudyPriority } from '../core/study_processor'; import { ItemAction, makeSummary, summaryToJson } from '../core/summary'; import { Study, Study_Channel, Study_Platform } from '../proto/generated/study'; import { VariationsSeed } from '../proto/generated/variations_seed'; import { storeDataToDirectory } from './tracker_lib'; -import JSON5 from 'json5'; function readDirectory(dir: string): Record { const files = fs @@ -40,10 +40,9 @@ test('seed serialization', async () => { isBraveSeed: true, }); - const serializedOutput = JSON5.stringify( - readDirectory(path.join(tempDir)), - { space: 2 }, - ); + const serializedOutput = JSON5.stringify(readDirectory(path.join(tempDir)), { + space: 2, + }); const serializedExpectations = fs .readFileSync('src/test/data/seed1.bin.processing_expectations') .toString(); From ac7aeb40f49eccfbbdb04754758dc79f37d93567 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 20 Nov 2024 17:29:02 +0700 Subject: [PATCH 13/19] Reuse channel/platform serializers --- src/core/serializers.ts | 34 ------------- src/seed_tools/utils/study_json_utils.test.ts | 4 +- src/seed_tools/utils/study_json_utils.ts | 49 +++++++++++-------- src/web/app/study_model.ts | 25 ++++++++-- 4 files changed, 51 insertions(+), 61 deletions(-) delete mode 100644 src/core/serializers.ts diff --git a/src/core/serializers.ts b/src/core/serializers.ts deleted file mode 100644 index 4da2cb09..00000000 --- a/src/core/serializers.ts +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) 2023 The Brave Authors. All rights reserved. -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. -import { Study_Channel, Study_Platform } from '../proto/generated/study'; - -export function getPlatformNameFromString(protoPlatfrom: string): string { - const PREFIX = 'PLATFORM_'; - if (protoPlatfrom.startsWith(PREFIX)) - return protoPlatfrom.substring(PREFIX.length); - return protoPlatfrom; -} - -export function getPlatfromName(protoPlatfrom: Study_Platform): string { - return getPlatformNameFromString(Study_Platform[protoPlatfrom]); -} - -export function getChannelNameFromString( - protoChannel: string, - isBraveSpecific: boolean, -): string { - if (isBraveSpecific) { - if (protoChannel === 'STABLE') return 'RELEASE'; - if (protoChannel === 'CANARY') return 'NIGHTLY'; - } - return protoChannel; -} - -export function getChannelName( - protoChannel: Study_Channel, - isBraveSpecific: boolean, -): string { - return getChannelNameFromString(Study_Channel[protoChannel], isBraveSpecific); -} diff --git a/src/seed_tools/utils/study_json_utils.test.ts b/src/seed_tools/utils/study_json_utils.test.ts index 9c29d313..6e3925e5 100644 --- a/src/seed_tools/utils/study_json_utils.test.ts +++ b/src/seed_tools/utils/study_json_utils.test.ts @@ -112,7 +112,7 @@ describe('stringifyStudies', () => { filter: { start_date: Math.floor(startDate.getTime() / 1000), channel: ['CANARY', 'BETA', 'STABLE'], - platform: ['PLATFORM_LINUX', 'PLATFORM_MAC'], + platform: ['LINUX', 'MAC'], }, }, { ignoreUnknownFields: false }, @@ -127,7 +127,7 @@ describe('stringifyStudies', () => { filter: { start_date: startDate.toISOString(), channel: ['CANARY', 'BETA', 'STABLE'], - platform: ['PLATFORM_LINUX', 'PLATFORM_MAC'], + platform: ['LINUX', 'MAC'], }, }, ]); diff --git a/src/seed_tools/utils/study_json_utils.ts b/src/seed_tools/utils/study_json_utils.ts index 4ddda910..ff77221d 100644 --- a/src/seed_tools/utils/study_json_utils.ts +++ b/src/seed_tools/utils/study_json_utils.ts @@ -86,6 +86,29 @@ export function stringifyStudies(studies: Study[], options?: Options): string { ); } +export function replaceChannels( + channels: string[] | undefined, + isChromium: boolean, +): string[] | undefined { + if (isChromium) return channels; + + return channels?.map((channel) => { + switch (channel) { + case 'CANARY': + return 'NIGHTLY'; + case 'STABLE': + return 'RELEASE'; + } + return channel; + }); +} + +export function replacePlatforms( + platforms: string[] | undefined, +): string[] | undefined { + return platforms?.map((platform) => platform.replace(/^PLATFORM_/, '')); +} + function jsonStudyReplacer( options: Options | undefined, key: string, @@ -96,26 +119,12 @@ function jsonStudyReplacer( case 'end_date': { return new Date(value * 1000).toISOString(); } - case 'channel': - if (options?.isChromium === true) { - return value; - } - return value.map((value: string): string => { - switch (value) { - case 'CANARY': - return 'NIGHTLY'; - case 'STABLE': - return 'RELEASE'; - } - return value; - }); - case 'platform': - if (options?.isChromium === true) { - return value; - } - return value.map((value: string): string => { - return value.replace(/^PLATFORM_/, ''); - }); + case 'channel': { + return replaceChannels(value, options?.isChromium === true); + } + case 'platform': { + return replacePlatforms(value); + } default: return value; } diff --git a/src/web/app/study_model.ts b/src/web/app/study_model.ts index 46b016e6..f7cd84f4 100644 --- a/src/web/app/study_model.ts +++ b/src/web/app/study_model.ts @@ -4,14 +4,21 @@ // You can obtain one at https://mozilla.org/MPL/2.0/. import { SeedType } from '../../core/base_types'; -import { getChannelName, getPlatfromName } from '../../core/serializers'; import { type ProcessedStudy, type StudyFilter, type StudyPriority, } from '../../core/study_processor'; import * as url_utils from '../../core/url_utils'; -import { Study_Filter } from '../../proto/generated/study'; +import { + Study_Channel, + Study_Filter, + Study_Platform, +} from '../../proto/generated/study'; +import { + replaceChannels, + replacePlatforms, +} from '../../seed_tools/utils/study_json_utils'; import { ExperimentModel } from './experiment_model'; export class StudyModel { @@ -55,12 +62,20 @@ export class StudyModel { } platforms(): string[] | undefined { - return this.filter()?.platform?.map((p) => getPlatfromName(p)); + const string_platforms = this.filter()?.platform?.map( + (p) => Study_Platform[p], + ); + return replacePlatforms(string_platforms); } channels(): string[] | undefined { - const isBraveSeed = this.seedType !== SeedType.UPSTREAM; - return this.filter()?.channel?.map((c) => getChannelName(c, isBraveSeed)); + const string_channels = this.filter()?.channel?.map( + (c) => Study_Channel[c], + ); + return replaceChannels( + string_channels, + this.seedType === SeedType.UPSTREAM, + ); } getConfigUrl(): string { From 623688370366c894aa59dc2b658a212cbd7c0b32 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Wed, 20 Nov 2024 17:55:23 +0700 Subject: [PATCH 14/19] move channel/platform functions to a new file --- src/seed_tools/utils/serializers.ts | 20 ++++++++++++++ src/seed_tools/utils/study_json_utils.ts | 34 +++++------------------- src/web/app/study_model.ts | 19 +++++-------- 3 files changed, 34 insertions(+), 39 deletions(-) create mode 100644 src/seed_tools/utils/serializers.ts diff --git a/src/seed_tools/utils/serializers.ts b/src/seed_tools/utils/serializers.ts new file mode 100644 index 00000000..46a86dbc --- /dev/null +++ b/src/seed_tools/utils/serializers.ts @@ -0,0 +1,20 @@ +// Copyright (c) 2024 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +export function channelToString(channel: string, isChromium: boolean): string { + if (isChromium) return channel; + + switch (channel) { + case 'CANARY': + return 'NIGHTLY'; + case 'STABLE': + return 'RELEASE'; + } + return channel; +} + +export function platformToString(platform: string): string { + return platform.replace(/^PLATFORM_/, ''); +} diff --git a/src/seed_tools/utils/study_json_utils.ts b/src/seed_tools/utils/study_json_utils.ts index ff77221d..032a147b 100644 --- a/src/seed_tools/utils/study_json_utils.ts +++ b/src/seed_tools/utils/study_json_utils.ts @@ -6,6 +6,7 @@ import { promises as fs } from 'fs'; import JSON5 from 'json5'; import { Study } from '../../proto/generated/study'; +import { channelToString, platformToString } from './serializers'; export interface Options { isChromium?: boolean; @@ -50,7 +51,7 @@ export function parseStudies( ): Study[] { const jsonStudies = JSON5.parse( studyArrayString, - jsonStudyReviever.bind(null, options), + jsonStudyReviewer.bind(null, options), ); if (!Array.isArray(jsonStudies)) { throw new Error('Root element must be an array'); @@ -86,29 +87,6 @@ export function stringifyStudies(studies: Study[], options?: Options): string { ); } -export function replaceChannels( - channels: string[] | undefined, - isChromium: boolean, -): string[] | undefined { - if (isChromium) return channels; - - return channels?.map((channel) => { - switch (channel) { - case 'CANARY': - return 'NIGHTLY'; - case 'STABLE': - return 'RELEASE'; - } - return channel; - }); -} - -export function replacePlatforms( - platforms: string[] | undefined, -): string[] | undefined { - return platforms?.map((platform) => platform.replace(/^PLATFORM_/, '')); -} - function jsonStudyReplacer( options: Options | undefined, key: string, @@ -120,17 +98,19 @@ function jsonStudyReplacer( return new Date(value * 1000).toISOString(); } case 'channel': { - return replaceChannels(value, options?.isChromium === true); + return value.map((c: string) => + channelToString(c, options?.isChromium === true), + ); } case 'platform': { - return replacePlatforms(value); + return value.map((p: string) => platformToString(p)); } default: return value; } } -function jsonStudyReviever( +function jsonStudyReviewer( options: Options | undefined, key: string, value: any, diff --git a/src/web/app/study_model.ts b/src/web/app/study_model.ts index f7cd84f4..bf0ac613 100644 --- a/src/web/app/study_model.ts +++ b/src/web/app/study_model.ts @@ -16,9 +16,9 @@ import { Study_Platform, } from '../../proto/generated/study'; import { - replaceChannels, - replacePlatforms, -} from '../../seed_tools/utils/study_json_utils'; + channelToString, + platformToString, +} from '../../seed_tools/utils/serializers'; import { ExperimentModel } from './experiment_model'; export class StudyModel { @@ -62,19 +62,14 @@ export class StudyModel { } platforms(): string[] | undefined { - const string_platforms = this.filter()?.platform?.map( - (p) => Study_Platform[p], + return this.filter()?.platform?.map((p) => + platformToString(Study_Platform[p]), ); - return replacePlatforms(string_platforms); } channels(): string[] | undefined { - const string_channels = this.filter()?.channel?.map( - (c) => Study_Channel[c], - ); - return replaceChannels( - string_channels, - this.seedType === SeedType.UPSTREAM, + return this.filter()?.channel?.map((c) => + channelToString(Study_Channel[c], this.seedType === SeedType.UPSTREAM), ); } From e05a32d97275a4d6103ec22faba0323b7f6330f5 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Thu, 21 Nov 2024 17:02:06 +0700 Subject: [PATCH 15/19] jsonStudyReviewer => jsonStudyReviever --- src/seed_tools/utils/study_json_utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/seed_tools/utils/study_json_utils.ts b/src/seed_tools/utils/study_json_utils.ts index 032a147b..ba5a9303 100644 --- a/src/seed_tools/utils/study_json_utils.ts +++ b/src/seed_tools/utils/study_json_utils.ts @@ -51,7 +51,7 @@ export function parseStudies( ): Study[] { const jsonStudies = JSON5.parse( studyArrayString, - jsonStudyReviewer.bind(null, options), + jsonStudyReviever.bind(null, options), ); if (!Array.isArray(jsonStudies)) { throw new Error('Root element must be an array'); @@ -110,7 +110,7 @@ function jsonStudyReplacer( } } -function jsonStudyReviewer( +function jsonStudyReviever( options: Options | undefined, key: string, value: any, From a60a242cc228d9404e6cb28b840bde3ecfdc4c28 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Thu, 21 Nov 2024 20:00:36 +0700 Subject: [PATCH 16/19] Review fixes --- src/core/url_utils.ts | 7 +++++-- src/finch_tracker/tracker_lib.ts | 7 +++++-- src/seed_tools/utils/study_json_utils.test.ts | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/core/url_utils.ts b/src/core/url_utils.ts index 4f838a84..62b55d37 100644 --- a/src/core/url_utils.ts +++ b/src/core/url_utils.ts @@ -46,8 +46,11 @@ export function getStudyRawConfigUrl( study: string, seedType: SeedType, ): string { - if (seedType === SeedType.UPSTREAM) - return `${getGitHubStorageUrl()}/blob/main/study/all-by-name/${study}`; + if (seedType === SeedType.UPSTREAM) { + return ( + getGitHubStorageUrl() + `/blob/main/study/all-by-name/${study}.json5` + ); + } return ( 'https://github.com/search?type=code' + '&q=repo%3Abrave%2Fbrave-variations' + diff --git a/src/finch_tracker/tracker_lib.ts b/src/finch_tracker/tracker_lib.ts index 6a752ebb..4afe3f34 100644 --- a/src/finch_tracker/tracker_lib.ts +++ b/src/finch_tracker/tracker_lib.ts @@ -33,8 +33,11 @@ export function groupStudies( const seed = VariationsSeed.fromBinary(seedData); const addStudy = (path: string, study: Study) => { const list = map[path]; - if (list !== undefined) list.push(study); - else map[path] = [study]; + if (list !== undefined) { + list.push(study); + } else { + map[path] = [study]; + } }; for (const study of seed.study) { diff --git a/src/seed_tools/utils/study_json_utils.test.ts b/src/seed_tools/utils/study_json_utils.test.ts index 6e3925e5..24adccf5 100644 --- a/src/seed_tools/utils/study_json_utils.test.ts +++ b/src/seed_tools/utils/study_json_utils.test.ts @@ -104,7 +104,7 @@ describe('stringifyStudies', () => { ]); }); - it('chromium mode should not modify channel, platform values', () => { + it('chromium mode should use chromium channel names', () => { const startDate = new Date('2022-01-01T00:00:00Z'); const study = Study.fromJson( { From 480ba381aec20bcf7ef297b093d74ecd8b7fe105 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Fri, 22 Nov 2024 15:16:43 +0700 Subject: [PATCH 17/19] fix typo --- src/seed_tools/utils/study_json_utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/seed_tools/utils/study_json_utils.ts b/src/seed_tools/utils/study_json_utils.ts index ba5a9303..c7bd059e 100644 --- a/src/seed_tools/utils/study_json_utils.ts +++ b/src/seed_tools/utils/study_json_utils.ts @@ -51,7 +51,7 @@ export function parseStudies( ): Study[] { const jsonStudies = JSON5.parse( studyArrayString, - jsonStudyReviever.bind(null, options), + jsonStudyReviver.bind(null, options), ); if (!Array.isArray(jsonStudies)) { throw new Error('Root element must be an array'); @@ -110,7 +110,7 @@ function jsonStudyReplacer( } } -function jsonStudyReviever( +function jsonStudyReviver( options: Options | undefined, key: string, value: any, From 5ef0b4e95c04499b7c0d4b55fdbaa9c382df5ae5 Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Fri, 22 Nov 2024 15:22:45 +0700 Subject: [PATCH 18/19] Group the logic for revive and replace --- .../utils/{serializers.ts => converters.ts} | 14 +++++++++++++ src/seed_tools/utils/study_json_utils.ts | 20 +++++-------------- src/web/app/study_model.ts | 2 +- 3 files changed, 20 insertions(+), 16 deletions(-) rename src/seed_tools/utils/{serializers.ts => converters.ts} (68%) diff --git a/src/seed_tools/utils/serializers.ts b/src/seed_tools/utils/converters.ts similarity index 68% rename from src/seed_tools/utils/serializers.ts rename to src/seed_tools/utils/converters.ts index 46a86dbc..7ec50342 100644 --- a/src/seed_tools/utils/serializers.ts +++ b/src/seed_tools/utils/converters.ts @@ -18,3 +18,17 @@ export function channelToString(channel: string, isChromium: boolean): string { export function platformToString(platform: string): string { return platform.replace(/^PLATFORM_/, ''); } + +export function stringToChannel(channel: string): string { + switch (channel) { + case 'NIGHTLY': + return 'CANARY'; + case 'RELEASE': + return 'STABLE'; + } + return channel; +} + +export function stringToPlatform(platform: string): string { + return `PLATFORM_${platform}`; +} diff --git a/src/seed_tools/utils/study_json_utils.ts b/src/seed_tools/utils/study_json_utils.ts index c7bd059e..2a71eaeb 100644 --- a/src/seed_tools/utils/study_json_utils.ts +++ b/src/seed_tools/utils/study_json_utils.ts @@ -6,7 +6,7 @@ import { promises as fs } from 'fs'; import JSON5 from 'json5'; import { Study } from '../../proto/generated/study'; -import { channelToString, platformToString } from './serializers'; +import * as converters from './converters'; export interface Options { isChromium?: boolean; @@ -99,11 +99,11 @@ function jsonStudyReplacer( } case 'channel': { return value.map((c: string) => - channelToString(c, options?.isChromium === true), + converters.channelToString(c, options?.isChromium === true), ); } case 'platform': { - return value.map((p: string) => platformToString(p)); + return value.map(converters.platformToString); } default: return value; @@ -132,22 +132,12 @@ function jsonStudyReviver( if (options?.isChromium === true) { return value; } - return value.map((value: string): string => { - switch (value) { - case 'NIGHTLY': - return 'CANARY'; - case 'RELEASE': - return 'STABLE'; - } - return value; - }); + return value.map(converters.stringToChannel); case 'platform': if (options?.isChromium === true) { return value; } - return value.map((value: string): string => { - return `PLATFORM_${value}`; - }); + return value.map(converters.stringToPlatform); default: return value; } diff --git a/src/web/app/study_model.ts b/src/web/app/study_model.ts index bf0ac613..2ec0f6f7 100644 --- a/src/web/app/study_model.ts +++ b/src/web/app/study_model.ts @@ -18,7 +18,7 @@ import { import { channelToString, platformToString, -} from '../../seed_tools/utils/serializers'; +} from '../../seed_tools/utils/converters'; import { ExperimentModel } from './experiment_model'; export class StudyModel { From 876216c7d901444c1bfd16ee68eaccc95a17995c Mon Sep 17 00:00:00 2001 From: Mikhail Atuchin Date: Fri, 22 Nov 2024 16:27:08 +0700 Subject: [PATCH 19/19] update package-lock.json --- package-lock.json | 542 ---------------------------------------------- 1 file changed, 542 deletions(-) diff --git a/package-lock.json b/package-lock.json index ae88d6c1..d5b15b77 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,8 +39,6 @@ "prettier": "3.3.3", "prettier-plugin-multiline-arrays": "3.0.6", "prettier-plugin-organize-imports": "3.2.4", - "protobufjs": "7.4.0", - "protobufjs-cli": "1.1.3", "style-loader": "3.3.4", "ts-jest": "29.2.5", "ts-loader": "9.5.1", @@ -1762,19 +1760,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@jsdoc/salty": { - "version": "0.2.8", - "resolved": "https://registry.npmjs.org/@jsdoc/salty/-/salty-0.2.8.tgz", - "integrity": "sha512-5e+SFVavj1ORKlKaKr2BmTOekmXbelU7dC0cDkQLqag7xfuTPuGMUFx7KWJuv4bYZrTsoL2Z18VVCOKYxzoHcg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "lodash": "^4.17.21" - }, - "engines": { - "node": ">=v12.0.0" - } - }, "node_modules/@leichtgewicht/ip-codec": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz", @@ -1903,80 +1888,6 @@ "@protobuf-ts/runtime": "^2.9.4" } }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", - "dev": true, - "license": "BSD-3-Clause" - }, "node_modules/@remix-run/router": { "version": "1.21.0", "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.21.0.tgz", @@ -2234,31 +2145,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/linkify-it": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-5.0.0.tgz", - "integrity": "sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/markdown-it": { - "version": "14.1.2", - "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-14.1.2.tgz", - "integrity": "sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/linkify-it": "^5", - "@types/mdurl": "^2" - } - }, - "node_modules/@types/mdurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-2.0.0.tgz", - "integrity": "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -3348,13 +3234,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/bluebird": { - "version": "3.7.2", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", - "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", - "dev": true, - "license": "MIT" - }, "node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -3578,19 +3457,6 @@ ], "license": "CC-BY-4.0" }, - "node_modules/catharsis": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.9.0.tgz", - "integrity": "sha512-prMTQVpcns/tzFgFVkVp6ak6RykZyWb3gu8ckUpd6YkTlacOd3DXGJjIpD4Q6zJirizvaiAjSSHlOsA+6sNh2A==", - "dev": true, - "license": "MIT", - "dependencies": { - "lodash": "^4.17.15" - }, - "engines": { - "node": ">= 10" - } - }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -4387,19 +4253,6 @@ "node": ">=10.13.0" } }, - "node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, "node_modules/envinfo": { "version": "7.13.0", "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.13.0.tgz", @@ -4679,93 +4532,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/escodegen": { - "version": "1.14.3", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz", - "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "esprima": "^4.0.1", - "estraverse": "^4.2.0", - "esutils": "^2.0.2", - "optionator": "^0.8.1" - }, - "bin": { - "escodegen": "bin/escodegen.js", - "esgenerate": "bin/esgenerate.js" - }, - "engines": { - "node": ">=4.0" - }, - "optionalDependencies": { - "source-map": "~0.6.1" - } - }, - "node_modules/escodegen/node_modules/estraverse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/escodegen/node_modules/levn": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", - "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", - "dev": true, - "license": "MIT", - "dependencies": { - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/escodegen/node_modules/optionator": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", - "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", - "dev": true, - "license": "MIT", - "dependencies": { - "deep-is": "~0.1.3", - "fast-levenshtein": "~2.0.6", - "levn": "~0.3.0", - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2", - "word-wrap": "~1.2.3" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/escodegen/node_modules/prelude-ls": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", - "dev": true, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/escodegen/node_modules/type-check": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", - "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", - "dev": true, - "license": "MIT", - "dependencies": { - "prelude-ls": "~1.1.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/eslint": { "version": "8.57.1", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", @@ -7480,56 +7246,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/js2xmlparser": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/js2xmlparser/-/js2xmlparser-4.0.2.tgz", - "integrity": "sha512-6n4D8gLlLf1n5mNLQPRfViYzu9RATblzPEtm1SthMX1Pjao0r9YI9nw7ZIfRxQMERS87mcswrg+r/OYrPRX6jA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "xmlcreate": "^2.0.4" - } - }, - "node_modules/jsdoc": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-4.0.3.tgz", - "integrity": "sha512-Nu7Sf35kXJ1MWDZIMAuATRQTg1iIPdzh7tqJ6jjvaU/GfDf+qi5UV8zJR3Mo+/pYFvm8mzay4+6O5EWigaQBQw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@babel/parser": "^7.20.15", - "@jsdoc/salty": "^0.2.1", - "@types/markdown-it": "^14.1.1", - "bluebird": "^3.7.2", - "catharsis": "^0.9.0", - "escape-string-regexp": "^2.0.0", - "js2xmlparser": "^4.0.2", - "klaw": "^3.0.0", - "markdown-it": "^14.1.0", - "markdown-it-anchor": "^8.6.7", - "marked": "^4.0.10", - "mkdirp": "^1.0.4", - "requizzle": "^0.2.3", - "strip-json-comments": "^3.1.0", - "underscore": "~1.13.2" - }, - "bin": { - "jsdoc": "jsdoc.js" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/jsdoc/node_modules/escape-string-regexp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", - "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -7619,16 +7335,6 @@ "node": ">=0.10.0" } }, - "node_modules/klaw": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/klaw/-/klaw-3.0.0.tgz", - "integrity": "sha512-0Fo5oir+O9jnXu5EefYbVK+mHMBeEVEy2cmctR1O1NECcCkPRreJKrS6Qt/j3KC2C148Dfo9i3pCmCMsdqGr0g==", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.1.9" - } - }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -7694,16 +7400,6 @@ "dev": true, "license": "MIT" }, - "node_modules/linkify-it": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", - "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "uc.micro": "^2.0.0" - } - }, "node_modules/lint-staged": { "version": "15.2.10", "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-15.2.10.tgz", @@ -7948,13 +7644,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true, - "license": "MIT" - }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -8080,13 +7769,6 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, - "node_modules/long": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", - "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==", - "dev": true, - "license": "Apache-2.0" - }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -8142,55 +7824,6 @@ "tmpl": "1.0.5" } }, - "node_modules/markdown-it": { - "version": "14.1.0", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", - "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1", - "entities": "^4.4.0", - "linkify-it": "^5.0.0", - "mdurl": "^2.0.0", - "punycode.js": "^2.3.1", - "uc.micro": "^2.1.0" - }, - "bin": { - "markdown-it": "bin/markdown-it.mjs" - } - }, - "node_modules/markdown-it-anchor": { - "version": "8.6.7", - "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-8.6.7.tgz", - "integrity": "sha512-FlCHFwNnutLgVTflOYHPW2pPcl2AACqVzExlkGQNsi4CJgqOHN7YTgDd4LuhgN1BFO3TS0vLAruV1Td6dwWPJA==", - "dev": true, - "license": "Unlicense", - "peerDependencies": { - "@types/markdown-it": "*", - "markdown-it": "*" - } - }, - "node_modules/marked": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", - "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", - "dev": true, - "license": "MIT", - "bin": { - "marked": "bin/marked.js" - }, - "engines": { - "node": ">= 12" - } - }, - "node_modules/mdurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", - "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", - "dev": true, - "license": "MIT" - }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -8344,29 +7977,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true, - "license": "MIT", - "bin": { - "mkdirp": "bin/cmd.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -9179,94 +8789,6 @@ "dev": true, "license": "MIT" }, - "node_modules/protobufjs": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", - "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==", - "dev": true, - "hasInstallScript": true, - "license": "BSD-3-Clause", - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/node": ">=13.7.0", - "long": "^5.0.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/protobufjs-cli": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/protobufjs-cli/-/protobufjs-cli-1.1.3.tgz", - "integrity": "sha512-MqD10lqF+FMsOayFiNOdOGNlXc4iKDCf0ZQPkPR+gizYh9gqUeGTWulABUCdI+N67w5RfJ6xhgX4J8pa8qmMXQ==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "chalk": "^4.0.0", - "escodegen": "^1.13.0", - "espree": "^9.0.0", - "estraverse": "^5.1.0", - "glob": "^8.0.0", - "jsdoc": "^4.0.0", - "minimist": "^1.2.0", - "semver": "^7.1.2", - "tmp": "^0.2.1", - "uglify-js": "^3.7.7" - }, - "bin": { - "pbjs": "bin/pbjs", - "pbts": "bin/pbts" - }, - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "protobufjs": "^7.0.0" - } - }, - "node_modules/protobufjs-cli/node_modules/glob": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", - "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", - "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^5.0.1", - "once": "^1.3.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/protobufjs-cli/node_modules/minimatch": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", - "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -9343,16 +8865,6 @@ "node": ">=6" } }, - "node_modules/punycode.js": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", - "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/pure-rand": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", @@ -9641,16 +9153,6 @@ "dev": true, "license": "MIT" }, - "node_modules/requizzle": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.4.tgz", - "integrity": "sha512-JRrFk1D4OQ4SqovXOgdav+K8EAhSB/LJZqCz8tbX0KObcdeM15Ss59ozWMBWmmINMagCwmqn4ZNryUGpBsl6Jw==", - "dev": true, - "license": "MIT", - "dependencies": { - "lodash": "^4.17.21" - } - }, "node_modules/resolve": { "version": "2.0.0-next.5", "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.5.tgz", @@ -10866,16 +10368,6 @@ "dev": true, "license": "MIT" }, - "node_modules/tmp": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", - "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.14" - } - }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", @@ -11192,26 +10684,6 @@ } } }, - "node_modules/uc.micro": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", - "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", - "dev": true, - "license": "MIT" - }, - "node_modules/uglify-js": { - "version": "3.19.2", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.2.tgz", - "integrity": "sha512-S8KA6DDI47nQXJSi2ctQ629YzwOVs+bQML6DAtvy0wgNdpi+0ySpQK0g2pxBq2xfF2z3YCscu7NNA8nXT9PlIQ==", - "dev": true, - "license": "BSD-2-Clause", - "bin": { - "uglifyjs": "bin/uglifyjs" - }, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", @@ -11228,13 +10700,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/underscore": { - "version": "1.13.7", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.7.tgz", - "integrity": "sha512-GMXzWtsc57XAtguZgaQViUOzs0KTkk8ojr3/xAxXLITqf/3EMwxC0inyETfDFjH/Krbhuep0HNbbjI9i/q3F3g==", - "dev": true, - "license": "MIT" - }, "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", @@ -11975,13 +11440,6 @@ } } }, - "node_modules/xmlcreate": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-2.0.4.tgz", - "integrity": "sha512-nquOebG4sngPmGPICTS5EnxqhKbCmz5Ox5hsszI2T6U5qdrJizBc+0ilYSEjTSzU0yZcmvppztXe/5Al5fUwdg==", - "dev": true, - "license": "Apache-2.0" - }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",