diff --git a/browsergym/experiments/requirements.txt b/browsergym/experiments/requirements.txt index 0a74745d..032e2abf 100644 --- a/browsergym/experiments/requirements.txt +++ b/browsergym/experiments/requirements.txt @@ -1,2 +1,3 @@ browsergym-core==0.8.1 tiktoken>=0.4 +dataclasses-json diff --git a/browsergym/experiments/src/bgym/__init__.py b/browsergym/experiments/src/bgym/__init__.py index 251d5b10..66b63d4e 100644 --- a/browsergym/experiments/src/bgym/__init__.py +++ b/browsergym/experiments/src/bgym/__init__.py @@ -1,12 +1,13 @@ +from browsergym.core.action.base import AbstractActionSet +from browsergym.core.action.highlevel import HighLevelActionSet +from browsergym.core.action.python import PythonActionSet from browsergym.experiments.agent import Agent, AgentInfo +from browsergym.experiments.benchmark import Benchmark, HighLevelActionSetArgs from browsergym.experiments.loop import ( + AbstractAgentArgs, EnvArgs, ExpArgs, - AbstractAgentArgs, ExpResult, StepInfo, StepTimestamps, ) -from browsergym.core.action.base import AbstractActionSet -from browsergym.core.action.highlevel import HighLevelActionSet -from browsergym.core.action.python import PythonActionSet diff --git a/browsergym/experiments/src/browsergym/experiments/benchmark.py b/browsergym/experiments/src/browsergym/experiments/benchmark.py new file mode 100644 index 00000000..95f72195 --- /dev/null +++ b/browsergym/experiments/src/browsergym/experiments/benchmark.py @@ -0,0 +1,376 @@ +import fnmatch +import io +import logging +import pkgutil +from dataclasses import dataclass, field +from typing import Literal, Optional + +import numpy as np +import pandas as pd +from dataclasses_json import DataClassJsonMixin, config + +from browsergym.core.action.highlevel import HighLevelActionSet +from browsergym.experiments.loop import SEED_MAX, EnvArgs + +logger = logging.getLogger(__name__) + + +@dataclass +class HighLevelActionSetArgs(DataClassJsonMixin): + subsets: list[HighLevelActionSet.ActionSubset] + # custom_actions: list[callable] | None # non-serializable argument, not supported + multiaction: bool + strict: bool + retry_with_force: bool + demo_mode: Literal["off", "default", "all_blue", "only_visible_elements"] + + def make_action_set(self): + return HighLevelActionSet( + subsets=self.subsets, + custom_actions=None, + multiaction=self.multiaction, + strict=self.strict, + retry_with_force=self.retry_with_force, + demo_mode=self.demo_mode, + ) + + +@dataclass +class Benchmark(DataClassJsonMixin): + name: str + high_level_action_set_args: HighLevelActionSetArgs + env_args_list: list[EnvArgs] + task_metadata: Optional[pd.DataFrame] = field( + default_factory=lambda: None, + metadata=config( + encoder=lambda df: df.to_dict(orient="records") if df is not None else None, + decoder=lambda items: pd.DataFrame(items) if items is not None else None, + ), + ) + + def __post_init__(self): + # if no metadata is present, generate a dataframe with single "task_name" column + if self.task_metadata is None: + unique_task_names = list(set([env_args.task_name for env_args in self.env_args_list])) + self.task_metadata = pd.DataFrame( + [{"task_name": task_name} for task_name in unique_task_names] + ) + # make sure all tasks in env_args are in the metadata + metadata_tasks = list(self.task_metadata["task_name"]) + assert all([env_args.task_name in metadata_tasks for env_args in self.env_args_list]) + + def subset_from_split(self, split: Literal["train", "eval", "test"]): + split_column = "split" + + # check for a split column in metadata + if not split_column in self.task_metadata.columns: + raise NotImplementedError( + f"This benchmark does not provide train/eval/test {split_column} (missing split column in task metadata)" + ) + + # recover the target split + sub_benchmark = self.subset_from_regexp(split_column, regexp=f"^{split}$") + + # check that the split exists (non-empty task list) + if not sub_benchmark.env_args_list: + raise ValueError(f"The {split} split for this benchmark is empty.") + + return sub_benchmark + + def subset_from_glob(self, column, glob): + subset = self.subset_from_regexp(column, regexp=fnmatch.translate(glob)) + subset.name = f"{self.name}[{column}={glob}]" + return subset + + def subset_from_regexp(self, column, regexp): + # extract the filtered task_name subset + task_name_subset = task_list_from_metadata(self.task_metadata, {column: regexp}) + + # return the sub benchmark + return Benchmark( + name=f"{self.name}[{column}=/{regexp}/]", + high_level_action_set_args=self.high_level_action_set_args, + env_args_list=[ + env_args + for env_args in self.env_args_list + if env_args.task_name in task_name_subset + ], + task_metadata=self.task_metadata, + ) + + +def task_metadata(benchmark_name: str): + return task_metadata_from_csv( + io.StringIO( + pkgutil.get_data(__name__, f"task_metadata/{benchmark_name}.csv").decode("utf-8") + ) + ) + + +def task_metadata_from_csv(filepath): + return pd.read_csv(filepath).fillna("") + + +def task_list_from_metadata(metadata: pd.DataFrame, filter: dict[str, str] = {}): + df = metadata + # filter the desired columns (AND filter) + for col_name, regex in filter.items(): + col_filter = df[col_name].astype(str).str.contains(regex, regex=True) + df = df[col_filter] + # return only the task names + return list(df["task_name"]) + + +# These are mean as the default highlevel action set to fairly evaluate agents on each benchmark. +# They are mostly arbitrary, the important thing is to evaluate different agents using the same action set for fairness. +DEFAULT_HIGHLEVEL_ACTION_SET_ARGS = { + # loosely from https://github.com/Farama-Foundation/miniwob-plusplus/blob/1bab0dffe34e92cc1049fe9443542029bf7e44a9/miniwob/action.py#L122 + "miniwob": HighLevelActionSetArgs( + subsets=["bid", "coord"], + multiaction=False, + strict=False, + retry_with_force=False, + demo_mode="off", + ), + # loosely from https://github.com/Farama-Foundation/miniwob-plusplus/blob/1bab0dffe34e92cc1049fe9443542029bf7e44a9/miniwob/action.py#L160 + "miniwob_bid": HighLevelActionSetArgs( + subsets=["bid"], + multiaction=False, + strict=False, + retry_with_force=False, + demo_mode="off", + ), + # loosely from https://github.com/Farama-Foundation/miniwob-plusplus/blob/1bab0dffe34e92cc1049fe9443542029bf7e44a9/miniwob/action.py#L173 + "miniwob_coord": HighLevelActionSetArgs( + subsets=["coord"], + multiaction=False, + strict=False, + retry_with_force=False, + demo_mode="off", + ), + "workarena_l1": HighLevelActionSetArgs( + subsets=["chat", "bid"], # no need for infeasible action + multiaction=False, + strict=False, + retry_with_force=False, + demo_mode="off", + ), + "workarena": HighLevelActionSetArgs( + subsets=["chat", "infeas", "bid"], + multiaction=False, + strict=False, + retry_with_force=False, + demo_mode="off", + ), + # from https://arxiv.org/abs/2307.13854 + "webarena": HighLevelActionSetArgs( + subsets=["chat", "infeas", "bid", "tab", "nav"], + multiaction=False, + strict=False, + retry_with_force=False, + demo_mode="off", + ), + # from https://arxiv.org/abs/2401.13649 + "visualwebarena": HighLevelActionSetArgs( + subsets=["chat", "infeas", "bid", "tab", "nav"], + multiaction=False, + strict=False, + retry_with_force=False, + demo_mode="off", + ), +} + +# all benchmarks are callables designed for lazy loading, i.e. `bench = BENCHMARKS["miniwob_all"]()` +BENCHMARKS = { + "miniwob_all": lambda: Benchmark( + name="miniwob_all", + high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["miniwob"], + env_args_list=_make_env_args_list_from_repeat_tasks( + task_list=task_list_from_metadata(metadata=task_metadata("miniwob")), + max_steps=10, + n_repeats=10, + seeds_rng=np.random.RandomState(42), + ), + task_metadata=task_metadata("miniwob"), + ), + "miniwob_webgum": lambda: Benchmark( + name="miniwob_webgum", + high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["miniwob"], + env_args_list=_make_env_args_list_from_repeat_tasks( + task_list=task_list_from_metadata( + metadata=task_metadata("miniwob"), filter={"webgum_subset": "True"} + ), + max_steps=10, + n_repeats=10, + seeds_rng=np.random.RandomState(42), + ), + task_metadata=task_metadata("miniwob"), + ), + "miniwob_tiny_test": lambda: Benchmark( + name="miniwob_tiny_test", + high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["miniwob"], + env_args_list=_make_env_args_list_from_repeat_tasks( + task_list=["miniwob.click-dialog", "miniwob.click-checkboxes"], + max_steps=5, + n_repeats=2, + seeds_rng=np.random.RandomState(42), + ), + task_metadata=task_metadata("miniwob"), + ), + "miniwob_train": lambda: Benchmark( + name="miniwob_train", + high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["miniwob"], + env_args_list=_make_env_args_list_from_repeat_tasks( + task_list=task_list_from_metadata( + metadata=task_metadata("miniwob"), + filter={"miniwob_category": "original|nodelay|debug|additional"}, + ), + max_steps=10, + n_repeats=10, + seeds_rng=np.random.RandomState(42), + ), + task_metadata=task_metadata("miniwob"), + ), + "miniwob_test": lambda: Benchmark( + name="miniwob_test", + high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["miniwob"], + env_args_list=_make_env_args_list_from_repeat_tasks( + task_list=task_list_from_metadata( + metadata=task_metadata("miniwob"), filter={"miniwob_category": "hidden test"} + ), + max_steps=10, + n_repeats=10, + seeds_rng=np.random.RandomState(42), + ), + task_metadata=task_metadata("miniwob"), + ), + "webarena": lambda: Benchmark( + name="webarena", + high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["webarena"], + env_args_list=_make_env_args_list_from_repeat_tasks( + task_list=task_list_from_metadata(metadata=task_metadata("webarena")), + max_steps=15, + n_repeats=1, + seeds_rng=np.random.RandomState(42), + ), + task_metadata=task_metadata("webarena"), + ), + "visualwebarena": lambda: Benchmark( + name="visualwebarena", + high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["visualwebarena"], + env_args_list=_make_env_args_list_from_repeat_tasks( + task_list=task_list_from_metadata(metadata=task_metadata("visualwebarena")), + max_steps=15, + n_repeats=1, + seeds_rng=np.random.RandomState(42), + ), + task_metadata=task_metadata("visualwebarena"), + ), + "workarena_l1": lambda: Benchmark( + name="workarena_l1", + high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["workarena_l1"], + env_args_list=_make_env_args_list_from_repeat_tasks( + task_list=task_list_from_metadata( + metadata=task_metadata("workarena"), filter={"level": "l1"} + ), + max_steps=15, + n_repeats=10, + seeds_rng=np.random.RandomState(42), + ), + task_metadata=task_metadata("workarena"), + ), + "workarena_l1_sort": lambda: Benchmark( + name="workarena_l1_sort", + high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["workarena_l1"], + env_args_list=_make_env_args_list_from_repeat_tasks( + task_list=task_list_from_metadata( + metadata=task_metadata("workarena"), filter={"level": "l1", "category": "list-sort"} + ), + max_steps=15, + n_repeats=10, + seeds_rng=np.random.RandomState(42), + ), + task_metadata=task_metadata("workarena"), + ), + "workarena_l2_agent_curriculum_eval": lambda: Benchmark( + name="workarena_l2_agent_curriculum_eval", + high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["workarena"], + env_args_list=_make_env_args_list_from_workarena_curriculum( + level="l2", + task_category_filter=None, + meta_seed=42, # meta seed for evaluation curriculum + max_steps=50, + curriculum_type="agent", + ), + task_metadata=task_metadata("workarena"), + ), + "workarena_l3_agent_curriculum_eval": lambda: Benchmark( + name="workarena_l3_agent_curriculum_eval", + high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["workarena"], + env_args_list=_make_env_args_list_from_workarena_curriculum( + level="l3", + task_category_filter=None, + meta_seed=42, # meta seed for evaluation curriculum + max_steps=50, + curriculum_type="agent", + ), + task_metadata=task_metadata("workarena"), + ), +} + + +def _make_env_args_list_from_workarena_curriculum( + level: Literal["l2", "l3"], + task_category_filter: str, + meta_seed: int, + max_steps: int, + curriculum_type: Literal["human", "agent"], +): + """ + Returns a WorkArena predefined task curriculum (e.g., task and seed combination). + """ + assert level in ("l2", "l3") + assert curriculum_type in ("human", "agent") + + env_args_list = [] + + from browsergym.workarena import get_all_tasks_agents + + all_task_tuples = get_all_tasks_agents( + filter=f"{level}.{task_category_filter}" if task_category_filter else level, + meta_seed=meta_seed, + is_agent_curriculum=(curriculum_type == "agent"), + ) + + for task, seed in all_task_tuples: + task_name = task.get_task_id() + env_args_list.append(EnvArgs(task_name=task_name, task_seed=seed, max_steps=max_steps)) + + return env_args_list + + +def _make_env_args_list_from_repeat_tasks( + task_list: list[str], max_steps: int, n_repeats: int, seeds_rng: np.random.RandomState +): + """ + Generates a list of `len(task_list)` time `n_repeats` environments arguments, using randomly generated seeds. + """ + env_args_list = [] + for task in task_list: + for seed in seeds_rng.randint(low=0, high=SEED_MAX, size=n_repeats): + env_args_list.append( + EnvArgs( + task_name=task, + task_seed=int(seed), + max_steps=max_steps, + headless=True, + record_video=False, + wait_for_user_message=False, + viewport=None, + slow_mo=None, + storage_state=None, + task_kwargs=None, + ) + ) + + return env_args_list diff --git a/browsergym/experiments/src/browsergym/experiments/loop.py b/browsergym/experiments/src/browsergym/experiments/loop.py index 95b81c5d..1eb47c68 100644 --- a/browsergym/experiments/src/browsergym/experiments/loop.py +++ b/browsergym/experiments/src/browsergym/experiments/loop.py @@ -17,6 +17,7 @@ import gymnasium as gym import numpy as np +from dataclasses_json import DataClassJsonMixin from PIL import Image from tqdm import tqdm @@ -27,9 +28,11 @@ logger = logging.getLogger(__name__) +SEED_MAX = 2 ^ 32 # arbitrary max value (exclusive), seems large enough + @dataclass -class EnvArgs: +class EnvArgs(DataClassJsonMixin): task_name: str task_seed: int = None max_steps: int = None @@ -152,7 +155,7 @@ def prepare(self, exp_root): This enables inspecting experiments that are not run yet. """ if self.env_args.task_seed is None: - self.env_args.task_seed = np.random.randint(0, 1000) + self.env_args.task_seed = np.random.randint(0, SEED_MAX) if self.exp_name is None: task_name = self.env_args.task_name diff --git a/browsergym/experiments/src/browsergym/experiments/task_metadata/miniwob.csv b/browsergym/experiments/src/browsergym/experiments/task_metadata/miniwob.csv new file mode 100644 index 00000000..0212e059 --- /dev/null +++ b/browsergym/experiments/src/browsergym/experiments/task_metadata/miniwob.csv @@ -0,0 +1,126 @@ +task_name,miniwob_category,comment,webgum_subset +miniwob.ascending-numbers,hidden test,,False +miniwob.bisect-angle,original,,False +miniwob.book-flight,original,delay,True +miniwob.book-flight-nodelay,nodelay,,False +miniwob.buy-ticket,hidden test,,False +miniwob.choose-date,original,delay,True +miniwob.choose-date-easy,debug,delay,True +miniwob.choose-date-medium,debug,delay,True +miniwob.choose-date-nodelay,nodelay,,False +miniwob.choose-list,original,,True +miniwob.circle-center,original,,False +miniwob.click-button,original,,True +miniwob.click-button-sequence,original,,True +miniwob.click-checkboxes,original,,True +miniwob.click-checkboxes-large,additional,,True +miniwob.click-checkboxes-soft,additional,,True +miniwob.click-checkboxes-transfer,additional,,True +miniwob.click-collapsible,original,delay,True +miniwob.click-collapsible-2,original,delay,True +miniwob.click-collapsible-2-nodelay,nodelay,,False +miniwob.click-collapsible-nodelay,nodelay,,False +miniwob.click-color,original,,True +miniwob.click-dialog,original,,True +miniwob.click-dialog-2,original,,True +miniwob.click-link,original,,True +miniwob.click-menu,original,,True +miniwob.click-menu-2,original,,False +miniwob.click-option,original,,True +miniwob.click-pie,original,delay,True +miniwob.click-pie-nodelay,nodelay,,False +miniwob.click-scroll-list,original,,True +miniwob.click-shades,original,,True +miniwob.click-shape,original,,True +miniwob.click-tab,original,,True +miniwob.click-tab-2,original,,True +miniwob.click-tab-2-easy,debug,,False +miniwob.click-tab-2-hard,additional,,True +miniwob.click-tab-2-medium,debug,,False +miniwob.click-test,original,,True +miniwob.click-test-2,original,,True +miniwob.click-test-transfer,debug,,False +miniwob.click-widget,original,,True +miniwob.copy-paste,original,,False +miniwob.copy-paste-2,original,,False +miniwob.count-shape,original,,True +miniwob.count-sides,original,,False +miniwob.daily-calendar,hidden test,,False +miniwob.drag-box,original,,False +miniwob.drag-circle,original,,False +miniwob.drag-cube,original,,False +miniwob.drag-items,original,,False +miniwob.drag-items-grid,original,,False +miniwob.drag-shapes,original,,False +miniwob.drag-shapes-2,hidden test,,False +miniwob.drag-single-shape,hidden test,,False +miniwob.drag-sort-numbers,original,,False +miniwob.draw-circle,hidden test,,False +miniwob.draw-line,hidden test,,False +miniwob.email-inbox,original,,True +miniwob.email-inbox-delete,debug,,False +miniwob.email-inbox-forward,debug,,False +miniwob.email-inbox-forward-nl,additional,,True +miniwob.email-inbox-forward-nl-turk,additional,,True +miniwob.email-inbox-important,debug,,False +miniwob.email-inbox-nl-turk,additional,,True +miniwob.email-inbox-noscroll,debug,,False +miniwob.email-inbox-reply,debug,,False +miniwob.email-inbox-star-reply,debug,,False +miniwob.enter-date,original,,True +miniwob.enter-password,original,,True +miniwob.enter-text,original,,True +miniwob.enter-text-2,original,,False +miniwob.enter-text-dynamic,original,,True +miniwob.enter-time,original,,True +miniwob.find-greatest,hidden test,,False +miniwob.find-midpoint,original,,False +miniwob.find-word,original,,False +miniwob.focus-text,original,,True +miniwob.focus-text-2,original,,True +miniwob.form-sequence,hidden test,,False +miniwob.form-sequence-2,hidden test,,False +miniwob.form-sequence-3,hidden test,,False +miniwob.generate-number,hidden test,,False +miniwob.grid-coordinate,original,,True +miniwob.guess-number,original,,True +miniwob.highlight-text,original,,False +miniwob.highlight-text-2,original,,False +miniwob.hot-cold,hidden test,,False +miniwob.identify-shape,original,,True +miniwob.login-user,original,,True +miniwob.login-user-popup,additional,,True +miniwob.multi-layouts,additional,,True +miniwob.multi-orderings,additional,,True +miniwob.navigate-tree,original,,True +miniwob.number-checkboxes,original,,False +miniwob.odd-or-even,hidden test,,False +miniwob.order-food,hidden test,,False +miniwob.phone-book,hidden test,,False +miniwob.read-table,original,,False +miniwob.read-table-2,original,,False +miniwob.resize-textarea,original,,False +miniwob.right-angle,original,,False +miniwob.scroll-text,original,,False +miniwob.scroll-text-2,original,,False +miniwob.search-engine,original,,True +miniwob.sign-agreement,hidden test,,False +miniwob.simple-algebra,original,,False +miniwob.simple-arithmetic,original,,False +miniwob.social-media,original,,True +miniwob.social-media-all,additional,,True +miniwob.social-media-some,additional,,True +miniwob.stock-market,hidden test,delay,False +miniwob.terminal,original,,False +miniwob.text-editor,original,,False +miniwob.text-transform,original,,False +miniwob.tic-tac-toe,original,,True +miniwob.unicode-test,debug,,False +miniwob.use-autocomplete,original,delay,True +miniwob.use-autocomplete-nodelay,nodelay,,False +miniwob.use-colorwheel,original,,False +miniwob.use-colorwheel-2,original,,False +miniwob.use-slider,original,,False +miniwob.use-slider-2,original,,False +miniwob.use-spinner,original,,True +miniwob.visual-addition,original,,False diff --git a/browsergym/experiments/src/browsergym/experiments/task_metadata/scripts.py b/browsergym/experiments/src/browsergym/experiments/task_metadata/scripts.py new file mode 100644 index 00000000..a97d8fb1 --- /dev/null +++ b/browsergym/experiments/src/browsergym/experiments/task_metadata/scripts.py @@ -0,0 +1,89 @@ +import importlib.resources +import json + + +def print_metadata_workarena(): + from browsergym.workarena import ( + AGENT_CURRICULUM_L2, + AGENT_CURRICULUM_L3, + TASK_CATEGORY_MAP, + ) + + metadata = [("task_name", "level", "category")] + + for task_name, category in TASK_CATEGORY_MAP.items(): + metadata.append((task_name, "l1", category)) + + for category, items in AGENT_CURRICULUM_L2.items(): + for task_set in items["buckets"]: + for task in task_set: + metadata.append((task.get_task_id(), "l2", category)) + + for category, items in AGENT_CURRICULUM_L3.items(): + for task_set in items["buckets"]: + for task in task_set: + metadata.append((task.get_task_id(), "l3", category)) + + print("\n".join([",".join(x) for x in metadata])) + + +def print_metadata_webarena(): + + import webarena + + metadata = [ + ( + "task_name", + "requires_reset", + "sites", + "eval_types", + ) + ] + all_configs_str = importlib.resources.files(webarena).joinpath("test.raw.json").read_text() + all_configs = json.loads(all_configs_str) + for task in all_configs: + metadata.append( + ( + f"webarena.{task['task_id']}", + str(task["require_reset"] == True), + " ".join(task["sites"]), + " ".join(task["eval"]["eval_types"]), + ) + ) + + print("\n".join([",".join(x) for x in metadata])) + + +def print_metadata_visualwebarena(): + import visualwebarena + + metadata = [ + ( + "task_name", + "requires_reset", + "sites", + "eval_types", + "reasoning_difficulty", + "visual_difficulty", + "overall_difficulty", + ) + ] + + all_configs_str = ( + importlib.resources.files(visualwebarena).joinpath("test_raw.json").read_text() + ) + all_configs = json.loads(all_configs_str) + for task in all_configs: + metadata.append( + ( + f"visualwebarena.{task['task_id']}", + str(task["require_reset"] == True), + " ".join(task["sites"]), + task["reasoning_difficulty"], + task["visual_difficulty"], + task["overall_difficulty"], + " ".join(task["eval"]["eval_types"]), + ) + ) + + print("\n".join([",".join(x) for x in metadata])) diff --git a/browsergym/experiments/src/browsergym/experiments/task_metadata/visualwebarena.csv b/browsergym/experiments/src/browsergym/experiments/task_metadata/visualwebarena.csv new file mode 100644 index 00000000..2f5dcfd3 --- /dev/null +++ b/browsergym/experiments/src/browsergym/experiments/task_metadata/visualwebarena.csv @@ -0,0 +1,911 @@ +task_name,requires_reset,sites,eval_types,reasoning_difficulty,visual_difficulty,overall_difficulty +visualwebarena.0,False,classifieds,medium,easy,medium,url_match +visualwebarena.1,False,classifieds,medium,easy,medium,url_match +visualwebarena.2,False,classifieds,medium,easy,medium,url_match +visualwebarena.3,False,classifieds,medium,easy,medium,url_match +visualwebarena.4,True,classifieds,hard,easy,hard,program_html +visualwebarena.5,True,classifieds,medium,easy,medium,program_html +visualwebarena.6,False,classifieds,medium,easy,medium,string_match +visualwebarena.7,False,classifieds,medium,easy,medium,string_match +visualwebarena.8,True,classifieds,hard,medium,hard,program_html page_image_query +visualwebarena.9,True,classifieds,hard,medium,hard,program_html page_image_query +visualwebarena.10,False,classifieds,easy,easy,easy,string_match +visualwebarena.11,False,classifieds,easy,easy,easy,string_match +visualwebarena.12,False,classifieds,easy,easy,easy,string_match +visualwebarena.13,False,classifieds,easy,easy,easy,string_match +visualwebarena.14,False,classifieds,easy,easy,easy,string_match +visualwebarena.15,False,classifieds,easy,easy,easy,string_match +visualwebarena.16,False,classifieds,easy,medium,medium,string_match +visualwebarena.17,False,classifieds,medium,easy,medium,url_match +visualwebarena.18,False,classifieds,medium,easy,medium,url_match +visualwebarena.19,False,classifieds,medium,easy,medium,url_match +visualwebarena.20,False,classifieds,medium,easy,medium,url_match +visualwebarena.21,False,classifieds,medium,easy,medium,url_match +visualwebarena.22,False,classifieds,easy,easy,easy,string_match +visualwebarena.23,False,classifieds,easy,easy,easy,string_match +visualwebarena.24,False,classifieds,easy,easy,easy,string_match +visualwebarena.25,False,classifieds,hard,easy,medium,string_match +visualwebarena.26,False,classifieds,hard,easy,medium,string_match +visualwebarena.27,False,classifieds,hard,medium,hard,string_match +visualwebarena.28,True,classifieds,hard,medium,hard,program_html +visualwebarena.29,True,classifieds,hard,easy,medium,program_html +visualwebarena.30,True,classifieds,hard,easy,medium,program_html +visualwebarena.31,True,classifieds,hard,easy,medium,program_html +visualwebarena.32,False,classifieds,hard,medium,medium,program_html +visualwebarena.33,False,classifieds,hard,easy,medium,program_html +visualwebarena.34,False,classifieds,hard,hard,hard,string_match +visualwebarena.35,False,classifieds,hard,easy,hard,string_match +visualwebarena.36,False,classifieds shopping,hard,easy,hard,string_match +visualwebarena.37,False,classifieds shopping,hard,medium,hard,string_match +visualwebarena.38,False,classifieds,hard,easy,hard,string_match +visualwebarena.39,False,classifieds,hard,easy,hard,string_match +visualwebarena.40,False,classifieds,medium,easy,medium,string_match +visualwebarena.41,False,classifieds,easy,easy,easy,string_match +visualwebarena.42,False,classifieds,easy,easy,easy,string_match +visualwebarena.43,False,classifieds,easy,easy,easy,string_match +visualwebarena.44,False,classifieds,hard,medium,medium,url_match +visualwebarena.45,False,classifieds,medium,medium,medium,url_match +visualwebarena.46,False,classifieds,hard,medium,hard,url_match +visualwebarena.47,False,classifieds,hard,medium,hard,url_match +visualwebarena.48,False,classifieds,medium,medium,medium,url_match +visualwebarena.49,False,classifieds,easy,easy,easy,string_match +visualwebarena.50,False,classifieds,easy,easy,easy,string_match +visualwebarena.51,False,classifieds,medium,hard,hard,url_match +visualwebarena.52,False,classifieds,easy,medium,easy,url_match +visualwebarena.53,False,classifieds,easy,easy,easy,url_match +visualwebarena.54,False,classifieds,easy,easy,easy,url_match +visualwebarena.55,False,classifieds,easy,easy,easy,url_match +visualwebarena.56,False,classifieds,easy,easy,easy,url_match +visualwebarena.57,True,classifieds,hard,easy,medium,program_html +visualwebarena.58,False,classifieds,medium,easy,medium,url_match +visualwebarena.59,False,classifieds,medium,medium,medium,url_match +visualwebarena.60,False,classifieds,hard,medium,medium,url_match +visualwebarena.61,False,classifieds,hard,medium,medium,url_match +visualwebarena.62,False,classifieds,hard,medium,medium,url_match +visualwebarena.63,False,classifieds,medium,hard,hard,url_match +visualwebarena.64,False,classifieds,medium,medium,medium,url_match +visualwebarena.65,False,classifieds,medium,medium,medium,url_match +visualwebarena.66,False,classifieds,medium,medium,medium,url_match +visualwebarena.67,False,classifieds,medium,medium,medium,url_match +visualwebarena.68,False,classifieds,medium,easy,easy,url_match +visualwebarena.69,False,classifieds,hard,hard,hard,url_match +visualwebarena.70,False,classifieds,medium,medium,medium,url_match +visualwebarena.71,False,classifieds,hard,medium,hard,url_match +visualwebarena.72,False,classifieds,hard,medium,hard,url_match +visualwebarena.73,False,classifieds,medium,medium,medium,url_match +visualwebarena.74,False,classifieds,medium,easy,easy,url_match +visualwebarena.75,False,classifieds,hard,easy,hard,program_html +visualwebarena.76,True,classifieds,hard,easy,hard,program_html +visualwebarena.77,True,classifieds,hard,easy,hard,program_html +visualwebarena.78,False,classifieds,easy,medium,medium,url_match +visualwebarena.79,False,classifieds,easy,medium,medium,url_match +visualwebarena.80,False,classifieds,easy,medium,medium,string_match +visualwebarena.81,False,classifieds,medium,hard,hard,url_match +visualwebarena.82,False,classifieds,medium,medium,medium,url_match string_match +visualwebarena.83,False,classifieds,easy,medium,medium,string_match +visualwebarena.84,False,classifieds,hard,medium,hard,url_match string_match +visualwebarena.85,False,classifieds,hard,medium,hard,url_match string_match +visualwebarena.86,False,classifieds,medium,easy,medium,url_match +visualwebarena.87,False,classifieds,easy,medium,easy,url_match +visualwebarena.88,False,classifieds,medium,medium,medium,url_match +visualwebarena.89,False,classifieds,medium,easy,easy,url_match +visualwebarena.90,False,classifieds,medium,medium,medium,url_match +visualwebarena.91,False,classifieds,medium,medium,medium,url_match +visualwebarena.92,False,classifieds,medium,hard,hard,url_match +visualwebarena.93,False,classifieds,easy,medium,medium,url_match +visualwebarena.94,False,classifieds,easy,easy,easy,url_match +visualwebarena.95,False,classifieds,hard,easy,hard,program_html +visualwebarena.96,False,classifieds,easy,medium,easy,url_match string_match +visualwebarena.97,False,classifieds,medium,easy,easy,url_match +visualwebarena.98,False,classifieds,medium,easy,easy,string_match +visualwebarena.99,False,classifieds,hard,hard,hard,url_match string_match +visualwebarena.100,False,classifieds,hard,hard,hard,string_match +visualwebarena.101,False,classifieds,medium,hard,hard,url_match +visualwebarena.102,False,classifieds,medium,hard,hard,string_match +visualwebarena.103,False,classifieds,medium,medium,medium,url_match +visualwebarena.104,False,classifieds,hard,hard,hard,program_html +visualwebarena.105,False,classifieds,medium,hard,hard,url_match +visualwebarena.106,False,classifieds,medium,medium,medium,string_match +visualwebarena.107,False,classifieds,medium,hard,hard,string_match +visualwebarena.108,False,classifieds,medium,medium,medium,string_match +visualwebarena.109,False,classifieds,medium,medium,medium,string_match +visualwebarena.110,False,classifieds,hard,hard,hard,string_match +visualwebarena.111,False,classifieds,medium,hard,hard,string_match +visualwebarena.112,False,classifieds,medium,hard,hard,url_match +visualwebarena.113,False,classifieds,hard,hard,hard,url_match +visualwebarena.114,False,classifieds,medium,medium,medium,url_match +visualwebarena.115,False,classifieds,medium,medium,medium,url_match +visualwebarena.116,False,classifieds,medium,medium,medium,url_match +visualwebarena.117,False,classifieds,medium,medium,medium,url_match +visualwebarena.118,False,classifieds,easy,medium,medium,string_match +visualwebarena.119,False,classifieds,easy,medium,medium,string_match +visualwebarena.120,False,classifieds,easy,medium,medium,string_match +visualwebarena.121,False,classifieds,easy,medium,medium,url_match +visualwebarena.122,False,classifieds,hard,medium,hard,url_match string_match +visualwebarena.123,False,classifieds,easy,medium,medium,url_match +visualwebarena.124,False,classifieds,easy,easy,easy,url_match +visualwebarena.125,False,classifieds,easy,easy,easy,url_match +visualwebarena.126,False,classifieds,medium,hard,hard,url_match +visualwebarena.127,False,classifieds,medium,hard,hard,url_match +visualwebarena.128,False,classifieds,easy,hard,medium,string_match +visualwebarena.129,False,classifieds,easy,medium,easy,string_match +visualwebarena.130,False,classifieds,medium,medium,medium,url_match +visualwebarena.131,False,classifieds,hard,hard,hard,url_match +visualwebarena.132,False,classifieds,easy,hard,medium,url_match +visualwebarena.133,False,classifieds,hard,medium,hard,program_html +visualwebarena.134,False,classifieds,easy,medium,medium,url_match +visualwebarena.135,False,classifieds,easy,medium,medium,string_match +visualwebarena.136,False,classifieds,hard,hard,hard,string_match +visualwebarena.137,False,classifieds,hard,hard,hard,string_match +visualwebarena.138,False,classifieds,hard,hard,hard,string_match +visualwebarena.139,False,classifieds,hard,hard,hard,url_match +visualwebarena.140,False,classifieds,medium,hard,hard,string_match +visualwebarena.141,False,classifieds,hard,hard,hard,url_match +visualwebarena.142,False,classifieds,hard,medium,hard,string_match +visualwebarena.143,True,classifieds,hard,hard,hard,program_html +visualwebarena.144,True,classifieds,hard,hard,hard,program_html +visualwebarena.145,True,classifieds,hard,hard,hard,program_html +visualwebarena.146,False,classifieds,easy,hard,medium,url_match +visualwebarena.147,False,classifieds,easy,hard,hard,url_match +visualwebarena.148,False,classifieds,easy,hard,medium,string_match +visualwebarena.149,False,classifieds,easy,hard,medium,string_match +visualwebarena.150,False,classifieds,easy,hard,medium,string_match +visualwebarena.151,False,classifieds,easy,easy,easy,url_match +visualwebarena.152,False,classifieds,easy,medium,medium,url_match +visualwebarena.153,False,classifieds,easy,hard,medium,url_match +visualwebarena.154,False,classifieds,medium,hard,hard,url_match +visualwebarena.155,False,classifieds,medium,hard,hard,url_match +visualwebarena.156,False,classifieds,medium,hard,hard,url_match +visualwebarena.157,False,classifieds,medium,medium,medium,url_match +visualwebarena.158,False,classifieds,medium,medium,medium,url_match +visualwebarena.159,True,classifieds,hard,medium,hard,program_html +visualwebarena.160,True,classifieds,hard,easy,hard,program_html +visualwebarena.161,False,classifieds,hard,mediun,hard,string_match +visualwebarena.162,False,classifieds,medium,medium,medium,url_match +visualwebarena.163,False,classifieds,easy,hard,medium,string_match +visualwebarena.164,False,classifieds,easy,easy,easy,string_match +visualwebarena.165,False,classifieds,easy,easy,easy,url_match +visualwebarena.166,False,classifieds,easy,easy,easy,url_match +visualwebarena.167,False,classifieds,easy,easy,easy,string_match +visualwebarena.168,False,classifieds,hard,medium,hard,url_match +visualwebarena.169,False,classifieds,hard,easy,medium,url_match +visualwebarena.170,False,classifieds,hard,hard,hard,url_match +visualwebarena.171,False,classifieds,medium,easy,easy,url_match +visualwebarena.172,False,classifieds,easy,easy,easy,url_match +visualwebarena.173,False,classifieds,easy,medium,easy,string_match +visualwebarena.174,False,classifieds,medium,hard,medium,string_match +visualwebarena.175,False,classifieds,hard,medium,hard,url_match +visualwebarena.176,False,classifieds,hard,medium,hard,url_match +visualwebarena.177,False,classifieds,hard,easy,hard,url_match +visualwebarena.178,False,classifieds,hard,medium,hard,string_match +visualwebarena.179,False,classifieds,medium,hard,hard,url_match +visualwebarena.180,False,classifieds,hard,hard,hard,program_html +visualwebarena.181,False,classifieds,medium,medium,medium,url_match +visualwebarena.182,False,classifieds,hard,hard,hard,url_match +visualwebarena.183,False,classifieds,hard,medium,hard,url_match +visualwebarena.184,False,classifieds,hard,hard,hard,url_match +visualwebarena.185,False,classifieds,hard,hard,hard,url_match +visualwebarena.186,False,classifieds,medium,medium,medium,string_match +visualwebarena.187,False,classifieds,easy,hard,medium,url_match +visualwebarena.188,False,classifieds,easy,medium,easy,url_match +visualwebarena.189,False,classifieds,easy,hard,medium,string_match +visualwebarena.190,False,classifieds,easy,hard,medium,url_match +visualwebarena.191,False,classifieds,easy,medium,medium,string_match +visualwebarena.192,False,classifieds,easy,hard,hard,string_match +visualwebarena.193,False,classifieds,easy,medium,easy,url_match +visualwebarena.194,False,classifieds,easy,hard,medium,string_match +visualwebarena.195,False,classifieds,medium,hard,hard,string_match +visualwebarena.196,False,classifieds,medium,hard,hard,string_match +visualwebarena.197,False,classifieds,medium,hard,hard,url_match +visualwebarena.198,False,classifieds,medium,hard,hard,url_match +visualwebarena.199,False,classifieds,easy,medium,medium,string_match +visualwebarena.200,False,classifieds,hard,hard,hard,url_match +visualwebarena.201,False,classifieds,hard,hard,hard,url_match +visualwebarena.202,False,classifieds,hard,hard,hard,url_match +visualwebarena.203,True,classifieds,medium,hard,hard,program_html +visualwebarena.204,False,classifieds,hard,hard,hard,url_match +visualwebarena.205,True,classifieds,medium,hard,hard,program_html +visualwebarena.206,False,classifieds,medium,medium,medium,url_match +visualwebarena.207,False,classifieds,hard,hard,hard,url_match +visualwebarena.208,True,classifieds,medium,medium,medium,program_html +visualwebarena.209,False,classifieds,easy,hard,medium,string_match +visualwebarena.210,False,classifieds,medium,medium,medium,url_match +visualwebarena.211,False,classifieds,hard,hard,hard,url_match +visualwebarena.212,False,classifieds,hard,hard,hard,url_match +visualwebarena.213,True,classifieds,hard,hard,hard,program_html +visualwebarena.214,False,classifieds,hard,hard,hard,url_match +visualwebarena.215,False,classifieds,hard,hard,hard,url_match +visualwebarena.216,False,classifieds,hard,hard,hard,url_match +visualwebarena.217,True,classifieds,hard,hard,hard,program_html +visualwebarena.218,False,classifieds,hard,hard,hard,url_match +visualwebarena.219,False,classifieds,hard,hard,hard,url_match +visualwebarena.220,False,classifieds,hard,hard,hard,string_match +visualwebarena.221,False,classifieds,easy,easy,easy,string_match +visualwebarena.222,False,classifieds,easy,hard,hard,string_match +visualwebarena.223,True,classifieds,medium,hard,hard,program_html +visualwebarena.224,False,classifieds,hard,hard,hard,url_match +visualwebarena.225,False,classifieds,hard,hard,hard,url_match +visualwebarena.226,False,classifieds,hard,hard,hard,url_match +visualwebarena.227,False,classifieds,hard,hard,hard,url_match +visualwebarena.228,False,classifieds,hard,hard,hard,program_html +visualwebarena.229,False,classifieds,hard,hard,hard,program_html +visualwebarena.230,False,classifieds,hard,hard,hard,url_match +visualwebarena.231,False,classifieds,hard,hard,hard,url_match +visualwebarena.232,False,classifieds,hard,hard,hard,url_match +visualwebarena.233,False,classifieds,easy,easy,easy,string_match +visualwebarena.234,False,reddit,easy,medium,easy,url_match +visualwebarena.235,False,reddit,medium,easy,easy,url_match +visualwebarena.236,False,reddit,medium,easy,medium,url_match +visualwebarena.237,False,reddit,medium,easy,easy,string_match +visualwebarena.238,False,reddit,medium,easy,easy,string_match +visualwebarena.239,False,reddit,medium,medium,medium,string_match +visualwebarena.240,False,reddit,hard,medium,hard,string_match +visualwebarena.241,False,reddit,hard,easy,hard,string_match +visualwebarena.242,False,reddit,hard,easy,hard,url_match +visualwebarena.243,False,reddit,medium,easy,medium,url_match +visualwebarena.244,False,reddit,hard,easy,hard,url_match +visualwebarena.245,False,reddit,hard,medium,hard,url_match +visualwebarena.246,False,reddit,hard,hard,hard,url_match +visualwebarena.247,False,reddit,hard,hard,hard,url_match +visualwebarena.248,False,reddit,medium,easy,medium,page_image_query +visualwebarena.249,False,reddit,medium,easy,medium,page_image_query +visualwebarena.250,False,reddit,hard,easy,hard,string_match +visualwebarena.251,False,reddit,medium,easy,medium,url_match +visualwebarena.252,False,reddit,medium,easy,medium,url_match +visualwebarena.253,False,reddit,hard,medium,hard,program_html +visualwebarena.254,False,reddit,hard,medium,hard,program_html +visualwebarena.255,False,reddit,hard,medium,hard,program_html +visualwebarena.256,False,reddit,hard,medium,hard,string_match +visualwebarena.257,False,reddit,hard,medium,hard,string_match +visualwebarena.258,False,reddit,hard,medium,hard,string_match +visualwebarena.259,False,reddit,hard,medium,hard,string_match +visualwebarena.260,False,reddit,hard,medium,hard,string_match +visualwebarena.261,False,reddit,hard,easy,hard,url_match page_image_query +visualwebarena.262,False,reddit,hard,easy,hard,program_html page_image_query +visualwebarena.263,False,reddit,hard,easy,hard,program_html page_image_query +visualwebarena.264,False,reddit,hard,medium,hard,url_match +visualwebarena.265,False,reddit,hard,medium,hard,string_match +visualwebarena.266,False,reddit,hard,medium,hard,url_match +visualwebarena.267,False,reddit,hard,hard,hard,url_match +visualwebarena.268,False,reddit,medium,medium,medium,url_match +visualwebarena.269,False,reddit,medium,medium,medium,url_match +visualwebarena.270,False,reddit,medium,medium,medium,url_match +visualwebarena.271,False,reddit,medium,medium,medium,url_match +visualwebarena.272,False,reddit,medium,hard,hard,url_match +visualwebarena.273,False,reddit,medium,easy,medium,string_match +visualwebarena.274,False,reddit,medium,medium,medium,url_match +visualwebarena.275,False,reddit,medium,medium,medium,url_match +visualwebarena.276,False,reddit,medium,medium,medium,url_match +visualwebarena.277,False,shopping reddit,hard,easy,hard,url_match program_html +visualwebarena.278,False,shopping reddit,hard,easy,hard,url_match program_html page_image_query +visualwebarena.279,False,wikipedia reddit,hard,hard,hard,string_match +visualwebarena.280,False,wikipedia reddit,hard,medium,hard,string_match +visualwebarena.281,False,wikipedia reddit,hard,medium,hard,string_match +visualwebarena.282,False,wikipedia reddit,hard,medium,hard,string_match +visualwebarena.283,False,wikipedia reddit,hard,hard,hard,string_match +visualwebarena.284,False,wikipedia reddit,medium,medium,medium,string_match +visualwebarena.285,False,wikipedia reddit,medium,medium,medium,string_match +visualwebarena.286,False,wikipedia reddit,medium,medium,medium,string_match +visualwebarena.287,False,wikipedia reddit,medium,hard,hard,string_match +visualwebarena.288,False,wikipedia reddit,medium,hard,hard,string_match +visualwebarena.289,False,wikipedia reddit,medium,hard,hard,string_match +visualwebarena.290,False,wikipedia reddit,medium,medium,medium,string_match +visualwebarena.291,False,wikipedia reddit,medium,hard,hard,string_match +visualwebarena.292,False,wikipedia reddit,medium,medium,medium,string_match +visualwebarena.293,False,wikipedia reddit,medium,hard,hard,string_match +visualwebarena.294,False,wikipedia reddit,medium,hard,hard,string_match +visualwebarena.295,False,wikipedia reddit,medium,hard,hard,string_match +visualwebarena.296,False,wikipedia reddit,medium,hard,hard,string_match +visualwebarena.297,False,wikipedia reddit,medium,hard,hard,string_match +visualwebarena.298,False,reddit,easy,hard,medium,string_match +visualwebarena.299,False,wikipedia reddit,medium,hard,hard,string_match +visualwebarena.300,False,wikipedia reddit,medium,hard,medium,string_match +visualwebarena.301,False,wikipedia reddit,medium,medium,medium,string_match +visualwebarena.302,False,wikipedia reddit,medium,medium,medium,string_match +visualwebarena.303,False,reddit,easy,medium,medium,program_html +visualwebarena.304,False,reddit,hard,medium,hard,string_match +visualwebarena.305,False,reddit,hard,medium,hard,program_html +visualwebarena.306,False,reddit,easy,medium,medium,program_html +visualwebarena.307,False,reddit,hard,medium,hard,program_html page_image_query url_match +visualwebarena.308,False,reddit,hard,hard,hard,program_html +visualwebarena.309,False,reddit shopping,hard,hard,hard,program_html +visualwebarena.310,False,reddit,easy,medium,medium,program_html +visualwebarena.311,False,reddit,easy,easy,easy,page_image_query +visualwebarena.312,False,reddit,hard,easy,medium,page_image_query +visualwebarena.313,False,reddit,medium,easy,medium,page_image_query +visualwebarena.314,False,reddit,easy,medium,medium,program_html +visualwebarena.315,False,reddit,easy,medium,medium,program_html +visualwebarena.316,False,reddit,easy,medium,medium,program_html +visualwebarena.317,False,reddit,easy,medium,medium,program_html +visualwebarena.318,False,reddit,medium,medium,medium,program_html +visualwebarena.319,False,reddit wikipedia,hard,hard,hard,program_html +visualwebarena.320,False,reddit,easy,medium,easy,string_match +visualwebarena.321,False,reddit,easy,medium,medium,string_match +visualwebarena.322,False,reddit,medium,medium,medium,program_html +visualwebarena.323,False,reddit,easy,easy,easy,program_html +visualwebarena.324,False,reddit,easy,easy,easy,program_html +visualwebarena.325,False,reddit,easy,easy,easy,program_html +visualwebarena.326,False,reddit,easy,easy,easy,string_match +visualwebarena.327,False,reddit,easy,easy,easy,string_match +visualwebarena.328,False,reddit,easy,easy,easy,string_match +visualwebarena.329,False,reddit,easy,easy,easy,string_match +visualwebarena.330,False,reddit,easy,easy,easy,string_match +visualwebarena.331,False,reddit,easy,easy,easy,string_match +visualwebarena.332,False,reddit,easy,easy,easy,page_image_query +visualwebarena.333,False,reddit,easy,easy,easy,page_image_query +visualwebarena.334,False,reddit,easy,easy,easy,page_image_query +visualwebarena.335,False,reddit,easy,easy,easy,page_image_query +visualwebarena.336,False,reddit,easy,medium,easy,program_html +visualwebarena.337,False,reddit,easy,easy,easy,program_html +visualwebarena.338,False,reddit,easy,easy,easy,program_html +visualwebarena.339,False,reddit,easy,easy,easy,program_html +visualwebarena.340,False,reddit,medium,easy,easy,program_html +visualwebarena.341,False,reddit,easy,easy,easy,program_html +visualwebarena.342,False,reddit,medium,easy,easy,program_html +visualwebarena.343,False,reddit,medium,easy,easy,program_html +visualwebarena.344,False,reddit,medium,easy,easy,program_html +visualwebarena.345,False,reddit,hard,hard,hard,page_image_query +visualwebarena.346,False,reddit,hard,hard,hard,page_image_query +visualwebarena.347,False,reddit,hard,hard,hard,page_image_query +visualwebarena.348,False,reddit,hard,easy,hard,url_match string_match +visualwebarena.349,False,reddit,hard,medium,hard,url_match string_match +visualwebarena.350,False,reddit,hard,easy,hard,url_match string_match +visualwebarena.351,False,wikipedia reddit,medium,medium,medium,string_match +visualwebarena.352,False,wikipedia reddit,medium,medium,medium,string_match +visualwebarena.353,False,wikipedia reddit,medium,easy,easy,string_match +visualwebarena.354,False,reddit,easy,easy,easy,string_match +visualwebarena.355,False,reddit,hard,easy,hard,url_match +visualwebarena.356,False,reddit,hard,easy,hard,url_match +visualwebarena.357,False,reddit,medium,easy,medium,string_match url_match +visualwebarena.358,False,reddit,medium,easy,medium,string_match +visualwebarena.359,False,reddit,medium,easy,medium,string_match url_match +visualwebarena.360,False,reddit,hard,easy,hard,url_match +visualwebarena.361,False,reddit,hard,easy,hard,url_match +visualwebarena.362,False,reddit,hard,easy,hard,url_match +visualwebarena.363,False,reddit,easy,easy,easy,program_html +visualwebarena.364,False,reddit,easy,easy,easy,program_html +visualwebarena.365,False,reddit,easy,easy,easy,program_html +visualwebarena.366,False,reddit,medium,easy,medium,string_match +visualwebarena.367,False,reddit,medium,easy,medium,string_match +visualwebarena.368,False,reddit,medium,easy,medium,string_match +visualwebarena.369,False,reddit,hard,easy,hard,url_match +visualwebarena.370,False,reddit,hard,easy,hard,url_match +visualwebarena.371,False,reddit,hard,easy,hard,url_match +visualwebarena.372,False,reddit,easy,medium,easy,program_html +visualwebarena.373,False,reddit,medium,easy,medium,program_html +visualwebarena.374,False,reddit,medium,easy,medium,program_html +visualwebarena.375,False,reddit,easy,easy,easy,program_html +visualwebarena.376,False,reddit,easy,easy,easy,program_html +visualwebarena.377,False,reddit,easy,easy,easy,program_html +visualwebarena.378,False,reddit,easy,medium,easy,program_html +visualwebarena.379,False,reddit,easy,medium,easy,program_html +visualwebarena.380,False,reddit,easy,medium,easy,program_html +visualwebarena.381,False,reddit,easy,easy,easy,program_html +visualwebarena.382,False,reddit,easy,easy,easy,program_html +visualwebarena.383,False,reddit,easy,easy,easy,program_html +visualwebarena.384,False,reddit,easy,easy,easy,program_html +visualwebarena.385,False,reddit,easy,easy,easy,program_html +visualwebarena.386,False,reddit,easy,easy,easy,program_html +visualwebarena.387,False,reddit,hard,medium,hard,url_match +visualwebarena.388,False,reddit,easy,medium,medium,string_match url_match +visualwebarena.389,False,reddit,hard,medium,hard,url_match +visualwebarena.390,False,reddit,hard,medium,medium,string_match +visualwebarena.391,False,reddit,easy,hard,medium,url_match +visualwebarena.392,True,reddit,hard,easy,medium,program_html +visualwebarena.393,True,reddit,hard,easy,medium,program_html +visualwebarena.394,True,reddit,medium,hard,medium,program_html +visualwebarena.395,False,reddit,hard,easy,medium,url_match +visualwebarena.396,False,reddit,medium,medium,medium,string_match +visualwebarena.397,False,reddit wikipedia,medium,medium,medium,string_match +visualwebarena.398,False,reddit wikipedia,medium,hard,medium,string_match +visualwebarena.399,False,reddit,hard,medium,medium,program_html +visualwebarena.400,False,reddit,hard,easy,medium,program_html +visualwebarena.401,False,reddit,medium,easy,medium,page_image_query +visualwebarena.402,True,reddit,medium,easy,medium,program_html page_image_query +visualwebarena.403,False,reddit,hard,easy,medium,program_html page_image_query +visualwebarena.404,True,reddit,medium,easy,medium,program_html +visualwebarena.405,True,reddit,medium,medium,medium,program_html +visualwebarena.406,True,reddit,medium,hard,medium,program_html +visualwebarena.407,True,reddit,medium,medium,medium,program_html +visualwebarena.408,True,reddit,medium,hard,medium,program_html +visualwebarena.409,False,reddit,hard,medium,medium,string_match +visualwebarena.410,True,reddit,medium,hard,hard,program_html +visualwebarena.411,True,reddit,medium,hard,medium,program_html +visualwebarena.412,True,reddit,medium,medium,medium,program_html +visualwebarena.413,False,reddit,medium,medium,medium,url_match +visualwebarena.414,False,reddit,easy,hard,medium,url_match +visualwebarena.415,False,reddit,hard,hard,hard,program_html +visualwebarena.416,True,reddit,hard,hard,hard,string_match +visualwebarena.417,False,reddit shopping,hard,hard,hard,program_html page_image_query +visualwebarena.418,False,reddit shopping,hard,easy,medium,program_html page_image_query +visualwebarena.419,False,reddit shopping wikipedia,hard,medium,hard,program_html page_image_query +visualwebarena.420,False,reddit shopping wikipedia,hard,medium,hard,program_html page_image_query +visualwebarena.421,False,reddit shopping wikipedia,hard,medium,hard,program_html page_image_query +visualwebarena.422,True,reddit,easy,easy,easy,program_html +visualwebarena.423,True,reddit,easy,easy,easy,program_html +visualwebarena.424,True,reddit,medium,easy,easy,program_html +visualwebarena.425,True,reddit,medium,hard,medium,string_match +visualwebarena.426,True,reddit,medium,medium,medium,string_match +visualwebarena.427,False,reddit,medium,easy,medium,program_html +visualwebarena.428,False,reddit,medium,hard,medium,program_html +visualwebarena.429,False,reddit,medium,hard,medium,program_html +visualwebarena.430,False,reddit,easy,easy,easy,url_match +visualwebarena.431,False,reddit,medium,easy,medium,url_match +visualwebarena.432,False,reddit,medium,medium,medium,program_html +visualwebarena.433,False,reddit,medium,hard,hard,program_html +visualwebarena.434,False,reddit,easy,easy,easy,url_match +visualwebarena.435,False,reddit,easy,easy,easy,url_match +visualwebarena.436,False,reddit,hard,hard,hard,program_html +visualwebarena.437,False,reddit,hard,hard,hard,program_html +visualwebarena.438,False,reddit,easy,easy,easy,url_match +visualwebarena.439,False,reddit,easy,medium,easy,string_match +visualwebarena.440,False,reddit,easy,easy,easy,string_match +visualwebarena.441,True,reddit shopping,hard,easy,hard,program_html page_image_query +visualwebarena.442,True,reddit shopping,hard,medium,hard,program_html page_image_query +visualwebarena.443,True,reddit shopping,hard,medium,hard,program_html page_image_query +visualwebarena.444,False,shopping,medium,easy,medium,program_html +visualwebarena.445,False,shopping,medium,easy,medium,program_html +visualwebarena.446,False,shopping,medium,easy,medium,program_html +visualwebarena.447,False,shopping,medium,easy,medium,url_match +visualwebarena.448,False,shopping,medium,easy,medium,url_match +visualwebarena.449,False,shopping,medium,easy,medium,url_match +visualwebarena.450,False,shopping,hard,easy,hard,string_match +visualwebarena.451,False,shopping,hard,medium,hard,string_match +visualwebarena.452,False,shopping,easy,easy,easy,string_match +visualwebarena.453,False,shopping,medium,easy,medium,url_match +visualwebarena.454,False,shopping,medium,easy,medium,url_match +visualwebarena.455,False,shopping,medium,easy,medium,url_match +visualwebarena.456,False,shopping,medium,easy,medium,url_match +visualwebarena.457,False,shopping,easy,easy,easy,string_match +visualwebarena.458,False,shopping,easy,easy,easy,string_match +visualwebarena.459,False,shopping,easy,easy,easy,string_match +visualwebarena.460,False,shopping,easy,easy,easy,string_match +visualwebarena.461,False,shopping,easy,easy,easy,string_match +visualwebarena.462,False,shopping,medium,easy,medium,string_match +visualwebarena.463,False,shopping,medium,easy,medium,string_match +visualwebarena.464,False,shopping,medium,easy,medium,string_match +visualwebarena.465,False,shopping,medium,easy,medium,string_match +visualwebarena.466,False,shopping,medium,easy,medium,string_match +visualwebarena.467,False,shopping,medium,easy,medium,string_match +visualwebarena.468,False,shopping,easy,easy,easy,url_match +visualwebarena.469,False,shopping,easy,easy,easy,url_match +visualwebarena.470,False,shopping,easy,easy,easy,url_match +visualwebarena.471,False,shopping,easy,easy,easy,url_match +visualwebarena.472,False,shopping,easy,easy,easy,url_match +visualwebarena.473,False,shopping,easy,easy,easy,url_match +visualwebarena.474,False,shopping,easy,medium,medium,url_match +visualwebarena.475,False,shopping,medium,easy,medium,program_html +visualwebarena.476,False,shopping,medium,easy,medium,program_html +visualwebarena.477,False,shopping,medium,easy,medium,string_match +visualwebarena.478,False,shopping,medium,easy,medium,string_match +visualwebarena.479,False,shopping,medium,easy,medium,string_match +visualwebarena.480,False,shopping,easy,easy,easy,program_html +visualwebarena.481,False,shopping,easy,easy,easy,program_html +visualwebarena.482,False,shopping,easy,easy,easy,program_html +visualwebarena.483,False,shopping,easy,easy,easy,program_html +visualwebarena.484,False,shopping,medium,medium,medium,string_match +visualwebarena.485,False,shopping,medium,easy,medium,string_match +visualwebarena.486,False,shopping,medium,easy,medium,string_match +visualwebarena.487,False,shopping,medium,medium,medium,url_match +visualwebarena.488,False,shopping,medium,easy,medium,url_match +visualwebarena.489,False,shopping,medium,medium,medium,url_match +visualwebarena.490,False,shopping,easy,easy,easy,program_html +visualwebarena.491,False,shopping,easy,easy,easy,program_html +visualwebarena.492,False,shopping,easy,easy,easy,program_html +visualwebarena.493,False,shopping,easy,medium,easy,program_html +visualwebarena.494,False,shopping,easy,easy,easy,program_html +visualwebarena.495,False,shopping,easy,medium,medium,program_html +visualwebarena.496,False,shopping,easy,easy,easy,program_html +visualwebarena.497,False,shopping,easy,easy,easy,program_html +visualwebarena.498,False,shopping,easy,medium,medium,program_html +visualwebarena.499,False,shopping,easy,easy,easy,program_html +visualwebarena.500,False,shopping,easy,easy,easy,program_html +visualwebarena.501,False,shopping,easy,hard,hard,program_html +visualwebarena.502,False,shopping,easy,easy,easy,program_html +visualwebarena.503,False,shopping,easy,easy,easy,string_match +visualwebarena.504,False,shopping,easy,easy,easy,string_match +visualwebarena.505,False,shopping,easy,easy,easy,string_match +visualwebarena.506,False,shopping,easy,easy,easy,string_match +visualwebarena.507,False,shopping,easy,easy,easy,string_match +visualwebarena.508,False,shopping,easy,easy,easy,string_match +visualwebarena.509,False,shopping,easy,easy,easy,string_match +visualwebarena.510,False,shopping,medium,easy,medium,string_match +visualwebarena.511,False,shopping,easy,easy,easy,string_match +visualwebarena.512,False,shopping,easy,easy,easy,string_match +visualwebarena.513,False,shopping,easy,easy,easy,string_match +visualwebarena.514,False,shopping,easy,easy,easy,string_match +visualwebarena.515,False,shopping,easy,easy,easy,string_match +visualwebarena.516,False,shopping,easy,easy,easy,program_html +visualwebarena.517,False,shopping,easy,easy,easy,program_html +visualwebarena.518,False,shopping,easy,easy,easy,program_html +visualwebarena.519,False,shopping,easy,easy,easy,program_html +visualwebarena.520,False,shopping,easy,easy,easy,program_html +visualwebarena.521,False,shopping,easy,easy,easy,program_html +visualwebarena.522,False,shopping,easy,easy,easy,program_html +visualwebarena.523,False,shopping,easy,easy,easy,string_match +visualwebarena.524,False,shopping,easy,easy,easy,string_match +visualwebarena.525,False,shopping,easy,easy,easy,string_match +visualwebarena.526,False,shopping,easy,medium,medium,program_html +visualwebarena.527,False,shopping,easy,medium,medium,program_html +visualwebarena.528,False,shopping,easy,medium,medium,program_html +visualwebarena.529,False,shopping,medium,medium,medium,program_html +visualwebarena.530,False,shopping,hard,easy,hard,program_html +visualwebarena.531,False,shopping,hard,easy,hard,program_html +visualwebarena.532,False,shopping,hard,hard,hard,program_html +visualwebarena.533,False,shopping,hard,medium,hard,program_html +visualwebarena.534,False,shopping,hard,medium,medium,string_match +visualwebarena.535,False,shopping,medium,medium,medium,string_match +visualwebarena.536,False,shopping,medium,medium,medium,string_match +visualwebarena.537,False,shopping,medium,medium,medium,program_html +visualwebarena.538,False,shopping,hard,hard,hard,program_html +visualwebarena.539,False,shopping,hard,medium,hard,program_html +visualwebarena.540,False,shopping,medium,easy,medium,program_html +visualwebarena.541,False,shopping,medium,easy,medium,program_html +visualwebarena.542,False,shopping,medium,easy,medium,program_html +visualwebarena.543,False,shopping,medium,medium,medium,program_html +visualwebarena.544,False,shopping,medium,medium,medium,program_html +visualwebarena.545,False,shopping,medium,easy,medium,program_html +visualwebarena.546,False,shopping,medium,easy,medium,program_html +visualwebarena.547,False,shopping,medium,medium,medium,program_html +visualwebarena.548,False,shopping,easy,medium,medium,program_html +visualwebarena.549,False,shopping,easy,medium,medium,program_html +visualwebarena.550,False,shopping,medium,medium,medium,program_html +visualwebarena.551,False,shopping,hard,hard,hard,program_html +visualwebarena.552,False,shopping,easy,easy,easy,page_image_query +visualwebarena.553,False,shopping,medium,easy,medium,program_html +visualwebarena.554,False,shopping,hard,hard,hard,program_html +visualwebarena.555,False,shopping,hard,hard,hard,program_html +visualwebarena.556,False,shopping,hard,hard,hard,program_html +visualwebarena.557,False,shopping,hard,hard,hard,program_html +visualwebarena.558,False,shopping,hard,hard,hard,program_html +visualwebarena.559,False,shopping,hard,hard,hard,program_html +visualwebarena.560,False,shopping,hard,hard,hard,program_html +visualwebarena.561,False,shopping,hard,hard,hard,program_html +visualwebarena.562,False,shopping,hard,hard,hard,program_html +visualwebarena.563,False,shopping,hard,hard,hard,program_html +visualwebarena.564,False,shopping,hard,hard,hard,program_html +visualwebarena.565,False,shopping,hard,medium,hard,string_match +visualwebarena.566,False,shopping,hard,medium,hard,string_match +visualwebarena.567,False,shopping,hard,medium,hard,string_match +visualwebarena.568,False,shopping,hard,medium,hard,string_match +visualwebarena.569,False,shopping,medium,hard,hard,string_match +visualwebarena.570,False,shopping,medium,hard,hard,string_match +visualwebarena.571,False,shopping,easy,hard,hard,string_match +visualwebarena.572,False,shopping,easy,hard,hard,string_match +visualwebarena.573,False,shopping,easy,hard,hard,string_match +visualwebarena.574,False,shopping,easy,medium,easy,string_match +visualwebarena.575,False,shopping,easy,medium,easy,string_match +visualwebarena.576,False,shopping,easy,medium,easy,string_match +visualwebarena.577,False,shopping,easy,medium,easy,string_match +visualwebarena.578,False,shopping,easy,medium,easy,string_match +visualwebarena.579,False,shopping,easy,medium,easy,string_match +visualwebarena.580,False,shopping,easy,medium,easy,string_match +visualwebarena.581,False,shopping,hard,hard,hard,string_match +visualwebarena.582,False,shopping,medium,medium,medium,string_match +visualwebarena.583,False,shopping,medium,hard,hard,string_match +visualwebarena.584,False,shopping,medium,medium,medium,string_match +visualwebarena.585,False,shopping,easy,medium,medium,string_match +visualwebarena.586,False,shopping,easy,medium,medium,string_match +visualwebarena.587,False,shopping,easy,hard,hard,string_match +visualwebarena.588,False,shopping,easy,medium,medium,string_match +visualwebarena.589,False,shopping,easy,medium,easy,string_match +visualwebarena.590,False,shopping,easy,medium,easy,string_match +visualwebarena.591,False,shopping,easy,hard,hard,string_match +visualwebarena.592,False,shopping,easy,easy,easy,string_match +visualwebarena.593,False,shopping,easy,easy,easy,string_match +visualwebarena.594,False,shopping,hard,easy,hard,url_match +visualwebarena.595,False,shopping,hard,easy,hard,url_match +visualwebarena.596,False,shopping,hard,medium,hard,url_match +visualwebarena.597,False,shopping,medium,medium,medium,program_html page_image_query +visualwebarena.598,False,shopping,medium,medium,medium,program_html page_image_query +visualwebarena.599,False,shopping,hard,medium,hard,url_match +visualwebarena.600,False,shopping,hard,medium,hard,url_match +visualwebarena.601,False,shopping,hard,medium,hard,url_match +visualwebarena.602,False,shopping,hard,hard,hard,program_html page_image_query +visualwebarena.603,False,shopping,easy,medium,medium,page_image_query +visualwebarena.604,False,shopping,medium,medium,medium,page_image_query +visualwebarena.605,False,shopping,medium,medium,medium,program_html +visualwebarena.606,False,shopping,medium,medium,medium,program_html +visualwebarena.607,False,shopping,medium,medium,medium,page_image_query +visualwebarena.608,False,shopping,hard,hard,hard,program_html page_image_query +visualwebarena.609,False,shopping,hard,hard,hard,program_html +visualwebarena.610,False,shopping,hard,hard,hard,program_html page_image_query +visualwebarena.611,False,shopping,hard,hard,hard,string_match +visualwebarena.612,False,shopping,medium,easy,easy,program_html +visualwebarena.613,False,shopping,hard,hard,hard,program_html +visualwebarena.614,False,shopping,hard,hard,hard,program_html +visualwebarena.615,False,shopping,hard,hard,hard,program_html +visualwebarena.616,False,shopping,medium,easy,easy,program_html +visualwebarena.617,False,shopping,medium,easy,easy,program_html +visualwebarena.618,False,shopping,hard,hard,hard,program_html +visualwebarena.619,False,shopping,hard,hard,hard,program_html +visualwebarena.620,False,shopping,medium,hard,hard,program_html +visualwebarena.621,False,shopping,medium,hard,hard,program_html +visualwebarena.622,False,shopping,hard,hard,hard,program_html +visualwebarena.623,False,shopping,hrad,hard,hard,program_html +visualwebarena.624,False,shopping,hard,hard,hard,program_html +visualwebarena.625,False,shopping,medium,medium,medium,program_html +visualwebarena.626,False,shopping,medium,medium,medium,program_html +visualwebarena.627,False,shopping,hard,hard,hard,program_html +visualwebarena.628,False,shopping,medium,easy,medium,program_html +visualwebarena.629,False,shopping,medium,easy,medium,program_html +visualwebarena.630,False,shopping,hard,medium,medium,program_html +visualwebarena.631,False,shopping,medium,easy,medium,program_html +visualwebarena.632,False,shopping,easy,easy,easy,string_match +visualwebarena.633,False,shopping,medium,medium,medium,program_html +visualwebarena.634,False,shopping,medium,easy,medium,program_html +visualwebarena.635,False,shopping,medium,hard,medium,program_html +visualwebarena.636,False,shopping,easy,hard,hard,string_match +visualwebarena.637,False,shopping,easy,easy,easy,string_match +visualwebarena.638,False,shopping,easy,hard,medium,string_match +visualwebarena.639,False,shopping,easy,easy,easy,string_match +visualwebarena.640,False,shopping,easy,easy,easy,program_html +visualwebarena.641,False,shopping,easy,medium,medium,program_html +visualwebarena.642,False,shopping,medium,medium,medium,program_html +visualwebarena.643,False,shopping,easy,easy,easy,string_match +visualwebarena.644,False,shopping,easy,medium,medium,string_match +visualwebarena.645,False,shopping,easy,easy,easy,string_match +visualwebarena.646,False,shopping,easy,medium,medium,string_match +visualwebarena.647,False,shopping,medium,medium,medium,string_match +visualwebarena.648,False,shopping,medium,medium,medium,string_match +visualwebarena.649,False,shopping,medium,medium,medium,string_match +visualwebarena.650,False,shopping,medium,medium,medium,string_match +visualwebarena.651,False,shopping,medium,hard,hard,string_match +visualwebarena.652,False,shopping,medium,hard,hard,string_match +visualwebarena.653,False,shopping,medium,hard,hard,string_match +visualwebarena.654,False,shopping,easy,medium,easy,string_match +visualwebarena.655,False,shopping,easy,medium,easy,string_match +visualwebarena.656,False,shopping,medium,medium,easy,program_html +visualwebarena.657,False,shopping,medium,hard,medium,program_html +visualwebarena.658,False,shopping,easy,medium,easy,program_html +visualwebarena.659,False,shopping,easy,medium,easy,program_html +visualwebarena.660,False,shopping,easy,medium,easy,program_html +visualwebarena.661,False,shopping,easy,medium,easy,string_match +visualwebarena.662,False,shopping,easy,medium,medium,string_match +visualwebarena.663,False,shopping,medium,hard,hard,string_match +visualwebarena.664,False,shopping,easy,hard,medium,string_match +visualwebarena.665,False,shopping,hard,easy,medium,program_html +visualwebarena.666,False,shopping,hard,easy,medium,program_html +visualwebarena.667,False,shopping,hard,easy,medium,program_html +visualwebarena.668,True,shopping,medium,easy,medium,program_html +visualwebarena.669,True,shopping,medium,hard,hard,program_html +visualwebarena.670,True,shopping,medium,medium,medium,program_html +visualwebarena.671,True,shopping,medium,hard,hard,program_html +visualwebarena.672,True,shopping,medium,easy,easy,program_html +visualwebarena.673,True,shopping,medium,hard,medium,program_html +visualwebarena.674,False,shopping,medium,medium,medium,program_html +visualwebarena.675,False,shopping,easy,easy,easy,program_html +visualwebarena.676,False,shopping,medium,medium,medium,program_html +visualwebarena.677,False,shopping,hard,medium,hard,program_html +visualwebarena.678,False,shopping,hard,medium,hard,program_html +visualwebarena.679,False,shopping,hard,medium,hard,program_html +visualwebarena.680,False,shopping,hard,easy,medium,program_html +visualwebarena.681,False,shopping,hard,easy,medium,program_html +visualwebarena.682,False,shopping,medium,easy,medium,program_html +visualwebarena.683,False,shopping,medium,easy,medium,program_html +visualwebarena.684,False,shopping,hard,medium,hard,program_html +visualwebarena.685,False,shopping,medium,hard,medium,string_match +visualwebarena.686,False,shopping,medium,hard,medium,string_match +visualwebarena.687,False,shopping,easy,hard,easy,string_match +visualwebarena.688,True,shopping,easy,easy,easy,program_html +visualwebarena.689,True,shopping,easy,medium,easy,program_html +visualwebarena.690,False,shopping,hard,medium,hard,program_html +visualwebarena.691,False,shopping,hard,hard,hard,program_html +visualwebarena.692,False,shopping,hard,easy,medium,program_html +visualwebarena.693,False,shopping,hard,easy,medium,program_html +visualwebarena.694,False,shopping,medium,easy,medium,program_html +visualwebarena.695,False,shopping,medium,easy,medium,program_html +visualwebarena.696,False,shopping,hard,easy,medium,program_html +visualwebarena.697,False,shopping,medium,medium,medium,string_match program_html +visualwebarena.698,False,shopping,medium,easy,medium,string_match program_html +visualwebarena.699,False,shopping,hard,medium,medium,string_match program_html +visualwebarena.700,False,shopping,hard,medium,hard,program_html +visualwebarena.701,False,shopping,hard,medium,hard,program_html +visualwebarena.702,False,shopping,medium,easy,medium,program_html +visualwebarena.703,False,shopping,hard,easy,medium,program_html +visualwebarena.704,False,shopping,medium,easy,medium,program_html +visualwebarena.705,False,shopping,medium,medium,medium,program_html +visualwebarena.706,False,shopping,medium,easy,medium,program_html +visualwebarena.707,False,shopping,hard,medium,medium,program_html +visualwebarena.708,False,shopping,medium,hard,medium,program_html +visualwebarena.709,False,shopping,hard,medium,hard,program_html +visualwebarena.710,False,shopping,hard,hard,hard,program_html +visualwebarena.711,True,shopping,medium,easy,easy,url_match program_html +visualwebarena.712,True,shopping,medium,easy,medium,url_match program_html +visualwebarena.713,True,shopping,hard,easy,medium,url_match program_html +visualwebarena.714,True,shopping,medium,medium,medium,url_match program_html +visualwebarena.715,True,shopping,hard,medium,hard,program_html +visualwebarena.716,True,shopping,hard,medium,hard,program_html +visualwebarena.717,True,shopping,medium,easy,easy,program_html +visualwebarena.718,False,shopping,medium,easy,easy,program_html +visualwebarena.719,False,shopping,hard,medium,hard,url_match +visualwebarena.720,False,shopping,hard,medium,hard,url_match +visualwebarena.721,False,shopping,medium,medium,medium,url_match +visualwebarena.722,False,shopping,hard,hard,hard,url_match +visualwebarena.723,False,shopping,hard,medium,medium,string_match +visualwebarena.724,False,shopping,medium,easy,medium,string_match +visualwebarena.725,False,shopping,medium,easy,easy,program_html +visualwebarena.726,False,shopping,medium,easy,easy,program_html +visualwebarena.727,False,shopping,medium,easy,medium,program_html +visualwebarena.728,False,shopping wikipedia,hard,easy,medium,program_html +visualwebarena.729,False,shopping,medium,easy,medium,program_html +visualwebarena.730,False,shopping,easy,easy,easy,page_image_query +visualwebarena.731,False,shopping,medium,easy,medium,string_match +visualwebarena.732,False,shopping,medium,easy,medium,program_html +visualwebarena.733,True,shopping,medium,easy,medium,program_html +visualwebarena.734,False,shopping,medium,easy,medium,string_match +visualwebarena.735,False,shopping,medium,medium,medium,string_match +visualwebarena.736,False,shopping,easy,medium,medium,string_match +visualwebarena.737,False,shopping,easy,medium,medium,string_match +visualwebarena.738,False,shopping,easy,medium,medium,string_match +visualwebarena.739,False,shopping,medium,medium,medium,page_image_query +visualwebarena.740,False,shopping,medium,easy,medium,program_html +visualwebarena.741,False,shopping,medium,easy,medium,program_html page_image_query +visualwebarena.742,False,shopping,easy,medium,medium,string_match +visualwebarena.743,False,shopping,easy,medium,medium,string_match +visualwebarena.744,False,shopping,easy,medium,medium,string_match +visualwebarena.745,False,shopping,easy,medium,medium,string_match +visualwebarena.746,False,shopping,easy,medium,medium,string_match +visualwebarena.747,False,shopping,easy,medium,medium,string_match +visualwebarena.748,False,shopping,easy,hard,hard,string_match +visualwebarena.749,False,shopping,easy,medium,medium,url_match +visualwebarena.750,False,shopping,easy,medium,medium,url_match +visualwebarena.751,False,shopping,easy,medium,medium,url_match +visualwebarena.752,False,shopping,easy,medium,medium,url_match +visualwebarena.753,False,shopping,easy,easy,easy,url_match +visualwebarena.754,False,shopping,easy,easy,easy,url_match +visualwebarena.755,False,shopping,easy,easy,easy,string_match +visualwebarena.756,False,shopping,easy,easy,easy,url_match +visualwebarena.757,False,shopping,medium,easy,easy,program_html +visualwebarena.758,False,shopping,easy,medium,easy,url_match +visualwebarena.759,False,shopping,easy,medium,easy,url_match +visualwebarena.760,False,shopping,easy,medium,medium,string_match +visualwebarena.761,False,shopping,easy,medium,medium,string_match +visualwebarena.762,False,shopping,easy,medium,medium,program_html +visualwebarena.763,False,shopping wikipedia,easy,medium,medium,program_html +visualwebarena.764,True,shopping,medium,medium,medium,program_html +visualwebarena.765,True,shopping,medium,medium,medium,program_html +visualwebarena.766,True,shopping,easy,easy,easy,string_match +visualwebarena.767,False,shopping,easy,easy,easy,string_match +visualwebarena.768,False,shopping,easy,easy,easy,string_match +visualwebarena.769,False,shopping,medium,easy,medium,program_html +visualwebarena.770,False,shopping,medium,easy,medium,program_html +visualwebarena.771,False,shopping,medium,easy,medium,program_html +visualwebarena.772,False,shopping,medium,medium,medium,program_html +visualwebarena.773,False,shopping,medium,medium,medium,program_html +visualwebarena.774,False,shopping,medium,medium,medium,program_html +visualwebarena.775,False,shopping,medium,medium,medium,program_html +visualwebarena.776,False,shopping,medium,medium,medium,program_html +visualwebarena.777,False,shopping,medium,easy,medium,string_match +visualwebarena.778,False,shopping,easy,easy,easy,string_match +visualwebarena.779,False,shopping,medium,medium,medium,program_html +visualwebarena.780,False,shopping,medium,hard,hard,program_html +visualwebarena.781,False,shopping,medium,hard,hard,string_match +visualwebarena.782,False,shopping,hard,medium,hard,program_html +visualwebarena.783,False,shopping,hard,medium,hard,program_html +visualwebarena.784,False,shopping,easy,easy,easy,string_match +visualwebarena.785,False,shopping,hard,hard,hard,program_html +visualwebarena.786,False,shopping,hard,hard,hard,program_html +visualwebarena.787,False,shopping,hard,hard,hard,program_html +visualwebarena.788,False,shopping,hard,hard,hard,program_html +visualwebarena.789,False,shopping wikipedia,medium,medium,medium,program_html +visualwebarena.790,False,shopping,hard,medium,hard,program_html +visualwebarena.791,False,shopping,hard,hard,hard,program_html +visualwebarena.792,False,shopping,medium,hard,hard,program_html +visualwebarena.793,False,shopping,hard,hard,hard,program_html +visualwebarena.794,False,shopping,medium,hard,hard,program_html +visualwebarena.795,False,shopping,medium,hard,hard,program_html +visualwebarena.796,False,shopping,easy,medium,medium,program_html +visualwebarena.797,False,shopping,easy,easy,easy,url_match +visualwebarena.798,False,shopping,hard,medium,medium,program_html +visualwebarena.799,False,shopping,easy,medium,medium,string_match +visualwebarena.800,False,shopping,medium,hard,hard,url_match +visualwebarena.801,False,shopping,easy,easy,easy,string_match +visualwebarena.802,False,shopping,easy,medium,medium,program_html +visualwebarena.803,False,shopping,easy,medium,medium,program_html +visualwebarena.804,False,shopping,easy,medium,medium,program_html +visualwebarena.805,False,shopping,easy,medium,medium,program_html +visualwebarena.806,False,shopping,easy,medium,medium,program_html +visualwebarena.807,False,shopping,easy,medium,medium,program_html +visualwebarena.808,False,shopping,easy,medium,medium,string_match +visualwebarena.809,False,shopping,medium,medium,medium,program_html +visualwebarena.810,False,shopping,medium,medium,medium,program_html +visualwebarena.811,False,shopping,easy,easy,easy,program_html +visualwebarena.812,False,shopping,easy,easy,easy,program_html +visualwebarena.813,False,shopping,hard,medium,hard,program_html +visualwebarena.814,False,shopping,hard,easy,hard,program_html +visualwebarena.815,False,shopping,medium,medium,medium,program_html +visualwebarena.816,False,shopping,hard,hard,hard,program_html +visualwebarena.817,False,shopping,medium,easy,medium,string_match +visualwebarena.818,False,shopping,medium,medium,medium,program_html +visualwebarena.819,False,shopping,medium,medium,medium,program_html +visualwebarena.820,False,shopping,medium,medium,medium,program_html +visualwebarena.821,False,shopping,medium,medium,medium,program_html +visualwebarena.822,False,shopping,easy,medium,medium,string_match +visualwebarena.823,False,shopping,easy,medium,medium,string_match +visualwebarena.824,False,shopping,easy,medium,medium,string_match +visualwebarena.825,False,shopping,hard,easy,medium,string_match +visualwebarena.826,False,shopping,easy,medium,medium,program_html +visualwebarena.827,False,shopping,easy,medium,medium,program_html +visualwebarena.828,False,shopping,medium,hard,hard,program_html +visualwebarena.829,False,shopping,medium,easy,medium,program_html +visualwebarena.830,False,shopping,hard,medium,hard,program_html +visualwebarena.831,False,shopping,medium,easy,medium,program_html +visualwebarena.832,False,shopping,medium,easy,easy,program_html +visualwebarena.833,False,shopping,medium,medium,medium,program_html +visualwebarena.834,False,shopping,hard,hard,hard,program_html +visualwebarena.835,False,shopping,hard,hard,hard,program_html +visualwebarena.836,False,shopping,medium,medium,medium,program_html +visualwebarena.837,False,shopping,easy,medium,medium,program_html +visualwebarena.838,False,shopping,easy,medium,medium,program_html +visualwebarena.839,False,shopping,easy,medium,medium,program_html +visualwebarena.840,False,shopping,easy,easy,easy,program_html +visualwebarena.841,False,shopping,easy,medium,medium,program_html +visualwebarena.842,False,shopping,easy,medium,medium,program_html +visualwebarena.843,False,shopping,medium,medium,medium,program_html +visualwebarena.844,False,shopping,easy,medium,medium,program_html +visualwebarena.845,False,shopping,easy,medium,medium,string_match +visualwebarena.846,False,shopping,easy,medium,medium,string_match +visualwebarena.847,False,shopping,easy,medium,medium,program_html +visualwebarena.848,False,shopping,easy,easy,easy,program_html +visualwebarena.849,False,shopping,hard,hard,hard,program_html +visualwebarena.850,False,shopping,hard,medium,hard,program_html +visualwebarena.851,False,shopping,medium,easy,medium,program_html +visualwebarena.852,False,shopping,hard,hard,hard,program_html +visualwebarena.853,False,shopping,hard,hard,hard,program_html +visualwebarena.854,False,shopping,hard,hard,hard,program_html +visualwebarena.855,False,shopping,hard,hard,hard,program_html +visualwebarena.856,False,shopping,hard,hard,hard,program_html +visualwebarena.857,False,shopping,hard,hard,hard,program_html +visualwebarena.858,False,shopping,hard,medium,hard,program_html +visualwebarena.859,False,shopping,hard,medium,hard,program_html +visualwebarena.860,False,shopping,medium,hard,hard,program_html +visualwebarena.861,False,shopping,medium,hard,hard,program_html +visualwebarena.862,False,shopping,medium,hard,hard,program_html +visualwebarena.863,False,shopping,medium,hard,hard,program_html +visualwebarena.864,False,shopping,medium,hard,hard,program_html +visualwebarena.865,False,shopping,medium,hard,hard,program_html +visualwebarena.866,False,shopping,medium,easy,medium,program_html +visualwebarena.867,False,shopping,easy,easy,easy,program_html +visualwebarena.868,False,shopping,hard,hard,hard,program_html +visualwebarena.869,False,shopping,hard,medium,hard,program_html +visualwebarena.870,False,shopping,hard,medium,hard,program_html +visualwebarena.871,False,shopping,easy,hard,medium,program_html +visualwebarena.872,False,shopping,easy,hard,medium,program_html +visualwebarena.873,False,shopping,easy,hard,medium,program_html +visualwebarena.874,False,shopping,hard,hard,hard,program_html +visualwebarena.875,False,shopping,hard,medium,hard,program_html +visualwebarena.876,False,shopping,medium,medium,medium,program_html +visualwebarena.877,False,shopping,hard,hard,hard,program_html +visualwebarena.878,False,shopping,hard,hard,hard,program_html +visualwebarena.879,False,shopping,hard,hard,hard,program_html +visualwebarena.880,False,shopping,hard,hard,hard,program_html +visualwebarena.881,False,shopping,hard,hard,hard,program_html +visualwebarena.882,False,shopping,hard,hard,hard,program_html +visualwebarena.883,False,shopping,hard,hard,hard,program_html +visualwebarena.884,False,shopping,hard,hard,hard,program_html +visualwebarena.885,False,shopping,hard,hard,hard,program_html +visualwebarena.886,False,shopping,hard,hard,hard,program_html +visualwebarena.887,False,shopping,hard,hard,hard,program_html +visualwebarena.888,False,shopping,hard,hard,hard,program_html +visualwebarena.889,False,shopping,easy,easy,easy,program_html +visualwebarena.890,False,shopping,easy,hard,medium,program_html +visualwebarena.891,False,shopping,easy,hard,medium,program_html +visualwebarena.892,False,shopping,easy,hard,medium,program_html +visualwebarena.893,False,shopping,easy,hard,medium,program_html +visualwebarena.894,False,shopping,hard,hard,hard,program_html +visualwebarena.895,False,shopping,hard,hard,hard,program_html +visualwebarena.896,False,shopping,hard,hard,hard,program_html +visualwebarena.897,False,shopping,medium,medium,medium,program_html +visualwebarena.898,False,shopping,hard,hard,hard,program_html +visualwebarena.899,False,shopping,hard,hard,hard,program_html +visualwebarena.900,False,shopping,hard,hard,hard,program_html +visualwebarena.901,False,shopping,hard,hard,hard,program_html +visualwebarena.902,False,shopping,hard,hard,hard,program_html +visualwebarena.903,False,shopping,hard,hard,hard,program_html +visualwebarena.904,False,shopping,hard,hard,hard,program_html +visualwebarena.905,False,shopping,hard,hard,hard,program_html +visualwebarena.906,False,shopping,hard,hard,hard,program_html +visualwebarena.907,False,shopping,easy,hard,hard,program_html +visualwebarena.908,False,shopping,easy,medium,medium,program_html +visualwebarena.909,False,shopping,easy,medium,medium,program_html diff --git a/browsergym/experiments/src/browsergym/experiments/task_metadata/webarena.csv b/browsergym/experiments/src/browsergym/experiments/task_metadata/webarena.csv new file mode 100644 index 00000000..dedcc1a2 --- /dev/null +++ b/browsergym/experiments/src/browsergym/experiments/task_metadata/webarena.csv @@ -0,0 +1,813 @@ +task_name,requires_reset,sites,eval_types +webarena.0,False,shopping_admin,string_match +webarena.1,False,shopping_admin,string_match +webarena.2,False,shopping_admin,string_match +webarena.3,False,shopping_admin,string_match +webarena.4,False,shopping_admin,string_match +webarena.5,False,shopping_admin,string_match +webarena.6,False,shopping_admin,string_match +webarena.7,False,map,string_match +webarena.8,False,map,string_match +webarena.9,False,map,string_match +webarena.10,False,map,string_match +webarena.11,False,shopping_admin,string_match +webarena.12,False,shopping_admin,string_match +webarena.13,False,shopping_admin,string_match +webarena.14,False,shopping_admin,string_match +webarena.15,False,shopping_admin,string_match +webarena.16,False,map,string_match +webarena.17,False,map,string_match +webarena.18,False,map,string_match +webarena.19,False,map,string_match +webarena.20,False,map,string_match +webarena.21,False,shopping,string_match +webarena.22,False,shopping,string_match +webarena.23,False,shopping,string_match +webarena.24,False,shopping,string_match +webarena.25,False,shopping,string_match +webarena.26,False,shopping,string_match +webarena.27,False,reddit,string_match +webarena.28,False,reddit,string_match +webarena.29,False,reddit,string_match +webarena.30,False,reddit,string_match +webarena.31,False,reddit,string_match +webarena.32,False,map,string_match +webarena.33,False,map,string_match +webarena.34,False,map,string_match +webarena.35,False,map,string_match +webarena.36,False,map,string_match +webarena.37,False,map,string_match +webarena.38,False,map,string_match +webarena.39,False,map,string_match +webarena.40,False,map,string_match +webarena.41,False,shopping_admin,string_match +webarena.42,False,shopping_admin,string_match +webarena.43,False,shopping_admin,string_match +webarena.44,False,gitlab,url_match +webarena.45,False,gitlab,url_match +webarena.46,False,gitlab,url_match +webarena.47,False,shopping,string_match +webarena.48,False,shopping,string_match +webarena.49,False,shopping,string_match +webarena.50,False,shopping,string_match +webarena.51,False,shopping,string_match +webarena.52,False,map,string_match +webarena.53,False,map,string_match +webarena.54,False,map,string_match +webarena.55,False,map,string_match +webarena.56,False,map,string_match +webarena.57,False,map,string_match +webarena.58,False,map,string_match +webarena.59,False,map,string_match +webarena.60,False,map,string_match +webarena.61,False,map,string_match +webarena.62,False,shopping_admin,string_match +webarena.63,False,shopping_admin,string_match +webarena.64,False,shopping_admin,string_match +webarena.65,False,shopping_admin,string_match +webarena.66,False,reddit,string_match +webarena.67,False,reddit,string_match +webarena.68,False,reddit,string_match +webarena.69,False,reddit,string_match +webarena.70,False,map,string_match +webarena.71,False,map,string_match +webarena.72,False,map,string_match +webarena.73,False,map,string_match +webarena.74,False,map,string_match +webarena.75,False,map,string_match +webarena.76,False,map,string_match +webarena.77,False,shopping_admin,string_match +webarena.78,False,shopping_admin,string_match +webarena.79,False,shopping_admin,string_match +webarena.80,False,map,string_match +webarena.81,False,map,string_match +webarena.82,False,map,string_match +webarena.83,False,map,string_match +webarena.84,False,map,string_match +webarena.85,False,map,string_match +webarena.86,False,map,string_match +webarena.87,False,map,string_match +webarena.88,False,map,string_match +webarena.89,False,map,string_match +webarena.90,False,map,string_match +webarena.91,False,map,string_match +webarena.92,False,map,string_match +webarena.93,False,map,string_match +webarena.94,False,shopping_admin,string_match +webarena.95,False,shopping_admin,string_match +webarena.96,False,shopping,string_match +webarena.97,False,map wikipedia,string_match +webarena.98,False,map,string_match +webarena.99,False,map,string_match +webarena.100,False,map,string_match +webarena.101,False,map,string_match +webarena.102,False,gitlab,url_match +webarena.103,False,gitlab,url_match +webarena.104,False,gitlab,url_match +webarena.105,False,gitlab,url_match +webarena.106,False,gitlab,url_match +webarena.107,False,shopping_admin,string_match +webarena.108,False,shopping_admin,string_match +webarena.109,False,shopping_admin,string_match +webarena.110,False,shopping_admin,string_match +webarena.111,False,shopping_admin,string_match +webarena.112,False,shopping_admin,string_match +webarena.113,False,shopping_admin,string_match +webarena.114,False,shopping_admin,string_match +webarena.115,False,shopping_admin,string_match +webarena.116,False,shopping_admin,string_match +webarena.117,False,shopping,string_match +webarena.118,False,shopping,program_html +webarena.119,False,shopping_admin,string_match +webarena.120,False,shopping_admin,string_match +webarena.121,False,shopping_admin,string_match +webarena.122,False,shopping_admin,string_match +webarena.123,False,shopping_admin,string_match +webarena.124,False,shopping,string_match +webarena.125,False,shopping,string_match +webarena.126,False,shopping,string_match +webarena.127,False,shopping_admin,string_match +webarena.128,False,shopping_admin,string_match +webarena.129,False,shopping_admin,string_match +webarena.130,False,shopping_admin,string_match +webarena.131,False,shopping_admin,string_match +webarena.132,False,gitlab,string_match +webarena.133,False,gitlab,string_match +webarena.134,False,gitlab,string_match +webarena.135,False,gitlab,string_match +webarena.136,False,gitlab,string_match +webarena.137,False,map,string_match +webarena.138,False,map,string_match +webarena.139,False,map,string_match +webarena.140,False,map,string_match +webarena.141,False,shopping,string_match +webarena.142,False,shopping,string_match +webarena.143,False,shopping,string_match +webarena.144,False,shopping,string_match +webarena.145,False,shopping,string_match +webarena.146,False,shopping,string_match +webarena.147,False,shopping,string_match +webarena.148,False,shopping,string_match +webarena.149,False,shopping,string_match +webarena.150,False,shopping,string_match +webarena.151,False,map,string_match +webarena.152,False,map,string_match +webarena.153,False,map,string_match +webarena.154,False,map,string_match +webarena.155,False,map,string_match +webarena.156,False,gitlab,url_match +webarena.157,False,shopping_admin,url_match +webarena.158,False,shopping,url_match +webarena.159,False,shopping,url_match +webarena.160,False,shopping,url_match +webarena.161,False,shopping,url_match +webarena.162,False,shopping,url_match +webarena.163,False,shopping,string_match +webarena.164,False,shopping,string_match +webarena.165,False,shopping,string_match +webarena.166,False,shopping,string_match +webarena.167,False,shopping,string_match +webarena.168,False,gitlab,string_match +webarena.169,False,gitlab,string_match +webarena.170,False,gitlab,string_match +webarena.171,False,gitlab,string_match +webarena.172,False,gitlab,string_match +webarena.173,False,gitlab,string_match url_match +webarena.174,False,gitlab,string_match url_match +webarena.175,False,gitlab,string_match url_match +webarena.176,False,gitlab,string_match url_match +webarena.177,False,gitlab,string_match url_match +webarena.178,False,gitlab,string_match url_match +webarena.179,False,gitlab,string_match url_match +webarena.180,False,gitlab,string_match url_match +webarena.181,False,gitlab,string_match url_match +webarena.182,False,gitlab,string_match url_match +webarena.183,False,shopping_admin,string_match +webarena.184,False,shopping_admin,string_match +webarena.185,False,shopping_admin,string_match +webarena.186,False,shopping_admin,string_match +webarena.187,False,shopping_admin,string_match +webarena.188,False,shopping,string_match +webarena.189,False,shopping,string_match +webarena.190,False,shopping,string_match +webarena.191,False,shopping,string_match +webarena.192,False,shopping,string_match +webarena.193,False,shopping_admin,string_match +webarena.194,False,shopping_admin,string_match +webarena.195,False,shopping_admin,string_match +webarena.196,False,shopping_admin,string_match +webarena.197,False,shopping_admin,string_match +webarena.198,False,shopping_admin,string_match +webarena.199,False,shopping_admin,string_match +webarena.200,False,shopping_admin,string_match +webarena.201,False,shopping_admin,string_match +webarena.202,False,shopping_admin,string_match +webarena.203,False,shopping_admin,string_match +webarena.204,False,shopping_admin,string_match +webarena.205,False,gitlab,string_match +webarena.206,False,gitlab,string_match +webarena.207,False,gitlab,string_match +webarena.208,False,shopping_admin,string_match +webarena.209,False,shopping_admin,string_match +webarena.210,False,shopping_admin,string_match +webarena.211,False,shopping_admin,string_match +webarena.212,False,shopping_admin,string_match +webarena.213,False,shopping_admin,string_match +webarena.214,False,shopping_admin,string_match +webarena.215,False,shopping_admin,string_match +webarena.216,False,shopping_admin,string_match +webarena.217,False,shopping_admin,string_match +webarena.218,False,map,string_match +webarena.219,False,map,string_match +webarena.220,False,map,string_match +webarena.221,False,map,string_match +webarena.222,False,map,string_match +webarena.223,False,map,string_match +webarena.224,False,map,string_match +webarena.225,False,shopping,string_match +webarena.226,False,shopping,string_match +webarena.227,False,shopping,string_match +webarena.228,False,shopping,string_match +webarena.229,False,shopping,string_match +webarena.230,False,shopping,string_match +webarena.231,False,shopping,string_match +webarena.232,False,shopping,string_match +webarena.233,False,shopping,string_match +webarena.234,False,shopping,string_match +webarena.235,False,shopping,string_match +webarena.236,False,map,string_match +webarena.237,False,map,string_match +webarena.238,False,shopping,url_match +webarena.239,False,shopping,url_match +webarena.240,False,shopping,url_match +webarena.241,False,shopping,url_match +webarena.242,False,shopping,url_match +webarena.243,False,shopping_admin,string_match +webarena.244,False,shopping_admin,string_match +webarena.245,False,shopping_admin,string_match +webarena.246,False,shopping_admin,string_match +webarena.247,False,shopping_admin,string_match +webarena.248,False,map,string_match +webarena.249,False,map,string_match +webarena.250,False,map,string_match +webarena.251,False,map,string_match +webarena.252,False,map,string_match +webarena.253,False,map,string_match +webarena.254,False,map,string_match +webarena.255,False,map,string_match +webarena.256,False,map,string_match +webarena.257,False,map,string_match +webarena.258,False,gitlab,url_match +webarena.259,False,gitlab,string_match +webarena.260,False,shopping,url_match +webarena.261,False,shopping,url_match +webarena.262,False,shopping,url_match +webarena.263,False,shopping,url_match +webarena.264,False,shopping,url_match +webarena.265,False,wikipedia map,string_match +webarena.266,False,wikipedia map,string_match +webarena.267,False,wikipedia map,string_match +webarena.268,False,wikipedia map,string_match +webarena.269,False,shopping,url_match +webarena.270,False,shopping,url_match +webarena.271,False,shopping,url_match +webarena.272,False,shopping,url_match +webarena.273,False,shopping,url_match +webarena.274,False,shopping,url_match +webarena.275,False,shopping,url_match +webarena.276,False,shopping,url_match +webarena.277,False,shopping,url_match +webarena.278,False,shopping,url_match +webarena.279,False,shopping,string_match +webarena.280,False,shopping,string_match +webarena.281,False,shopping,string_match +webarena.282,False,shopping,string_match +webarena.283,False,shopping,url_match +webarena.284,False,shopping,url_match +webarena.285,False,shopping,url_match +webarena.286,False,shopping,url_match +webarena.287,False,map,string_match +webarena.288,False,shopping_admin,string_match +webarena.289,False,shopping_admin,string_match +webarena.290,False,shopping_admin,string_match +webarena.291,False,shopping_admin,string_match +webarena.292,False,shopping_admin,string_match +webarena.293,False,gitlab,string_match +webarena.294,False,gitlab,string_match +webarena.295,False,gitlab,string_match +webarena.296,False,gitlab,string_match +webarena.297,False,gitlab,string_match +webarena.298,False,shopping,url_match +webarena.299,False,shopping,url_match +webarena.300,False,shopping,url_match +webarena.301,False,shopping,string_match +webarena.302,False,shopping,string_match +webarena.303,False,gitlab,string_match +webarena.304,False,gitlab,string_match +webarena.305,False,gitlab,string_match +webarena.306,False,gitlab,string_match +webarena.307,False,gitlab,string_match +webarena.308,False,gitlab,string_match +webarena.309,False,gitlab,string_match +webarena.310,False,gitlab,string_match +webarena.311,False,gitlab,string_match +webarena.312,False,gitlab,string_match +webarena.313,False,shopping,string_match +webarena.314,False,gitlab,string_match +webarena.315,False,gitlab,string_match +webarena.316,False,gitlab,string_match +webarena.317,False,gitlab,string_match +webarena.318,False,gitlab,string_match +webarena.319,False,shopping,string_match +webarena.320,False,shopping,string_match +webarena.321,False,shopping,string_match +webarena.322,False,shopping,string_match +webarena.323,False,shopping,string_match +webarena.324,False,shopping,url_match +webarena.325,False,shopping,url_match +webarena.326,False,shopping,url_match +webarena.327,False,shopping,url_match +webarena.328,False,shopping,url_match +webarena.329,False,shopping,string_match +webarena.330,False,shopping,string_match +webarena.331,False,shopping,string_match +webarena.332,False,shopping,string_match +webarena.333,False,shopping,string_match +webarena.334,False,shopping,string_match +webarena.335,False,shopping,string_match +webarena.336,False,shopping,string_match +webarena.337,False,shopping,string_match +webarena.338,False,shopping,string_match +webarena.339,False,gitlab,url_match +webarena.340,False,gitlab,url_match +webarena.341,False,gitlab,url_match +webarena.342,False,gitlab,url_match +webarena.343,False,gitlab,url_match +webarena.344,False,shopping_admin,string_match +webarena.345,False,shopping_admin,string_match +webarena.346,False,shopping_admin,string_match +webarena.347,False,shopping_admin,string_match +webarena.348,False,shopping_admin,string_match +webarena.349,False,gitlab,string_match +webarena.350,False,gitlab,string_match +webarena.351,False,shopping,url_match +webarena.352,False,shopping,url_match +webarena.353,False,shopping,url_match +webarena.354,False,shopping,url_match +webarena.355,False,shopping,url_match +webarena.356,False,map,program_html +webarena.357,False,gitlab,url_match +webarena.358,False,shopping,string_match +webarena.359,False,shopping,string_match +webarena.360,False,shopping,string_match +webarena.361,False,shopping,string_match +webarena.362,False,shopping,string_match +webarena.363,False,map,string_match +webarena.364,False,map,string_match +webarena.365,False,map,string_match +webarena.366,False,map,string_match +webarena.367,False,map,string_match +webarena.368,False,shopping,string_match +webarena.369,False,map,program_html +webarena.370,False,map,program_html +webarena.371,False,map,program_html +webarena.372,False,map,program_html +webarena.373,False,map,program_html +webarena.374,False,shopping_admin,url_match +webarena.375,False,shopping_admin,url_match +webarena.376,False,shopping,string_match +webarena.377,False,map,url_match +webarena.378,False,map,url_match +webarena.379,False,map,url_match +webarena.380,False,map,url_match +webarena.381,False,map,url_match +webarena.382,False,map,string_match +webarena.383,False,map,string_match +webarena.384,False,shopping,string_match +webarena.385,False,shopping,string_match +webarena.386,False,shopping,string_match +webarena.387,False,shopping,string_match +webarena.388,False,shopping,string_match +webarena.389,False,gitlab,program_html +webarena.390,False,gitlab,program_html +webarena.391,False,gitlab,program_html +webarena.392,False,gitlab,program_html +webarena.393,False,gitlab,program_html +webarena.394,False,gitlab,program_html +webarena.395,False,gitlab,program_html +webarena.396,False,gitlab,program_html +webarena.397,False,gitlab,program_html +webarena.398,False,gitlab,program_html +webarena.399,False,reddit,program_html +webarena.400,False,reddit,program_html +webarena.401,False,reddit,program_html +webarena.402,False,reddit,program_html +webarena.403,False,reddit,program_html +webarena.404,False,reddit,program_html +webarena.405,False,reddit,program_html +webarena.406,False,reddit,program_html +webarena.407,False,reddit,program_html +webarena.408,False,reddit,program_html +webarena.409,False,reddit,program_html +webarena.410,False,reddit,program_html +webarena.411,False,gitlab,program_html +webarena.412,False,gitlab,program_html +webarena.413,False,gitlab,program_html +webarena.414,False,gitlab,program_html +webarena.415,False,gitlab,program_html +webarena.416,False,gitlab,program_html +webarena.417,False,gitlab,program_html +webarena.418,False,gitlab,program_html +webarena.419,False,gitlab,program_html +webarena.420,False,gitlab,program_html +webarena.421,False,gitlab,program_html +webarena.422,False,gitlab,program_html +webarena.423,False,shopping_admin,program_html +webarena.424,False,wikipedia map,program_html +webarena.425,False,wikipedia map,program_html +webarena.426,False,wikipedia map,program_html +webarena.427,False,wikipedia map,program_html +webarena.428,False,wikipedia map,program_html +webarena.429,False,wikipedia map,program_html +webarena.430,False,wikipedia map,program_html +webarena.431,False,shopping,program_html +webarena.432,False,shopping,program_html +webarena.433,False,shopping,program_html +webarena.434,False,shopping,program_html +webarena.435,False,shopping,program_html +webarena.436,False,shopping,program_html +webarena.437,False,shopping,program_html +webarena.438,False,shopping,program_html +webarena.439,False,shopping,program_html +webarena.440,False,shopping,program_html +webarena.441,False,gitlab,program_html +webarena.442,False,gitlab,program_html +webarena.443,False,gitlab,program_html +webarena.444,False,gitlab,program_html +webarena.445,False,gitlab,program_html +webarena.446,False,gitlab,program_html +webarena.447,False,gitlab,program_html +webarena.448,False,gitlab,program_html +webarena.449,False,gitlab,program_html +webarena.450,False,gitlab,program_html +webarena.451,False,gitlab,program_html +webarena.452,False,gitlab,program_html +webarena.453,False,shopping_admin,program_html +webarena.454,False,shopping_admin,program_html +webarena.455,False,shopping_admin,program_html +webarena.456,False,shopping_admin,program_html +webarena.457,False,shopping_admin,program_html +webarena.458,False,shopping_admin,program_html +webarena.459,False,shopping_admin,program_html +webarena.460,False,shopping_admin,program_html +webarena.461,False,shopping_admin,program_html +webarena.462,False,shopping_admin,program_html +webarena.463,False,shopping_admin,program_html +webarena.464,False,shopping_admin,program_html +webarena.465,False,shopping,program_html +webarena.466,False,shopping,program_html +webarena.467,False,shopping,program_html +webarena.468,False,shopping,program_html +webarena.469,False,shopping,program_html +webarena.470,False,shopping_admin,program_html +webarena.471,False,shopping_admin,program_html +webarena.472,False,shopping_admin,program_html +webarena.473,False,shopping_admin,program_html +webarena.474,False,shopping_admin,program_html +webarena.475,False,gitlab,program_html +webarena.476,False,gitlab,program_html +webarena.477,False,gitlab,program_html +webarena.478,False,gitlab,program_html +webarena.479,False,gitlab,program_html +webarena.480,False,gitlab,program_html +webarena.481,False,gitlab,program_html +webarena.482,False,gitlab,program_html +webarena.483,False,gitlab,program_html +webarena.484,False,gitlab,program_html +webarena.485,False,gitlab,program_html +webarena.486,False,shopping_admin,program_html +webarena.487,False,shopping_admin,program_html +webarena.488,False,shopping_admin,program_html +webarena.489,False,shopping_admin,program_html +webarena.490,False,shopping_admin,program_html +webarena.491,False,shopping_admin,string_match +webarena.492,False,shopping_admin,program_html +webarena.493,False,shopping_admin,program_html +webarena.494,False,shopping_admin,program_html +webarena.495,False,shopping_admin,program_html +webarena.496,False,shopping_admin,program_html +webarena.497,False,shopping_admin,program_html +webarena.498,False,shopping_admin,program_html +webarena.499,False,shopping_admin,program_html +webarena.500,False,shopping_admin,program_html +webarena.501,False,shopping_admin,program_html +webarena.502,False,shopping_admin,program_html +webarena.503,False,shopping_admin,program_html +webarena.504,False,shopping_admin,program_html +webarena.505,False,shopping_admin,program_html +webarena.506,False,shopping,program_html +webarena.507,False,shopping,program_html +webarena.508,False,shopping,program_html +webarena.509,False,shopping,program_html +webarena.510,False,shopping,program_html +webarena.511,False,shopping,program_html +webarena.512,False,shopping,program_html +webarena.513,False,shopping,program_html +webarena.514,False,shopping,program_html +webarena.515,False,shopping,program_html +webarena.516,False,shopping,program_html +webarena.517,False,shopping,program_html +webarena.518,False,shopping,program_html +webarena.519,False,shopping,program_html +webarena.520,False,shopping,program_html +webarena.521,False,shopping,program_html +webarena.522,False,gitlab,program_html +webarena.523,False,gitlab,program_html +webarena.524,False,gitlab,program_html +webarena.525,False,gitlab,program_html +webarena.526,False,gitlab,program_html +webarena.527,False,gitlab,program_html +webarena.528,False,shopping,program_html +webarena.529,False,shopping,program_html +webarena.530,False,shopping,program_html +webarena.531,False,shopping,program_html +webarena.532,False,shopping,program_html +webarena.533,False,gitlab,program_html +webarena.534,False,gitlab,program_html +webarena.535,False,gitlab,program_html +webarena.536,False,gitlab,program_html +webarena.537,False,gitlab,program_html +webarena.538,False,shopping_admin,program_html +webarena.539,False,shopping_admin,program_html +webarena.540,False,shopping_admin,program_html +webarena.541,False,shopping_admin,program_html +webarena.542,False,shopping_admin,program_html +webarena.543,False,shopping_admin,program_html +webarena.544,False,shopping_admin,program_html +webarena.545,False,shopping_admin,program_html +webarena.546,False,shopping_admin,program_html +webarena.547,False,shopping_admin,program_html +webarena.548,False,shopping_admin,program_html +webarena.549,False,shopping_admin,program_html +webarena.550,False,shopping_admin,program_html +webarena.551,False,shopping_admin,program_html +webarena.552,False,gitlab reddit,program_html +webarena.553,False,gitlab reddit,program_html +webarena.554,False,gitlab reddit,program_html +webarena.555,False,gitlab reddit,program_html +webarena.556,False,gitlab wikipedia,program_html +webarena.557,False,gitlab wikipedia,program_html +webarena.558,False,gitlab wikipedia,program_html +webarena.559,False,gitlab wikipedia,program_html +webarena.560,False,gitlab wikipedia,program_html +webarena.561,False,gitlab wikipedia,program_html +webarena.562,False,gitlab reddit,program_html +webarena.563,False,gitlab reddit,program_html +webarena.564,False,gitlab reddit,program_html +webarena.565,False,gitlab reddit,program_html +webarena.566,False,gitlab reddit,program_html +webarena.567,False,gitlab,program_html +webarena.568,False,gitlab,program_html +webarena.569,False,gitlab,program_html +webarena.570,False,gitlab,program_html +webarena.571,False,shopping,program_html +webarena.572,False,shopping,program_html +webarena.573,False,shopping,program_html +webarena.574,False,shopping,program_html +webarena.575,False,shopping,program_html +webarena.576,False,gitlab,program_html +webarena.577,False,gitlab,program_html +webarena.578,False,gitlab,program_html +webarena.579,False,gitlab,program_html +webarena.580,False,reddit,program_html +webarena.581,False,reddit,program_html +webarena.582,False,reddit,program_html +webarena.583,False,reddit,program_html +webarena.584,False,reddit,program_html +webarena.585,False,shopping,program_html +webarena.586,False,shopping,program_html +webarena.587,False,shopping,program_html +webarena.588,False,shopping,program_html +webarena.589,False,shopping,program_html +webarena.590,False,gitlab,url_match program_html +webarena.591,False,gitlab,url_match program_html +webarena.592,False,gitlab,url_match program_html +webarena.593,False,gitlab,url_match program_html +webarena.594,False,gitlab,url_match program_html +webarena.595,False,reddit,url_match program_html +webarena.596,False,reddit,url_match program_html +webarena.597,False,reddit,url_match program_html +webarena.598,False,reddit,url_match program_html +webarena.599,False,reddit,url_match program_html +webarena.600,False,reddit,url_match program_html +webarena.601,False,reddit,url_match program_html +webarena.602,False,reddit,url_match program_html +webarena.603,False,reddit,url_match program_html +webarena.604,False,reddit,url_match program_html +webarena.605,False,reddit,url_match program_html +webarena.606,False,reddit,url_match program_html +webarena.607,False,reddit,url_match program_html +webarena.608,False,reddit,url_match program_html +webarena.609,False,reddit,url_match program_html +webarena.610,False,reddit,url_match program_html +webarena.611,False,reddit,url_match program_html +webarena.612,False,reddit,url_match program_html +webarena.613,False,reddit,url_match program_html +webarena.614,False,reddit,url_match program_html +webarena.615,False,reddit,url_match program_html +webarena.616,False,reddit,url_match program_html +webarena.617,False,reddit,url_match program_html +webarena.618,False,reddit,url_match program_html +webarena.619,False,reddit,url_match program_html +webarena.620,False,reddit,url_match program_html +webarena.621,False,reddit,url_match program_html +webarena.622,False,reddit,url_match program_html +webarena.623,False,reddit,url_match program_html +webarena.624,False,reddit,url_match program_html +webarena.625,False,reddit,url_match program_html +webarena.626,False,reddit,url_match program_html +webarena.627,False,reddit,url_match program_html +webarena.628,False,reddit,url_match program_html +webarena.629,False,reddit,url_match program_html +webarena.630,False,reddit,url_match program_html +webarena.631,False,reddit,url_match program_html +webarena.632,False,reddit,url_match program_html +webarena.633,False,reddit,url_match program_html +webarena.634,False,reddit,url_match program_html +webarena.635,False,reddit,url_match program_html +webarena.636,False,reddit,url_match program_html +webarena.637,False,reddit,url_match program_html +webarena.638,False,reddit,url_match program_html +webarena.639,False,reddit,url_match program_html +webarena.640,False,reddit,url_match program_html +webarena.641,False,reddit,url_match program_html +webarena.642,False,reddit,url_match program_html +webarena.643,False,reddit,url_match program_html +webarena.644,False,reddit,url_match program_html +webarena.645,False,reddit,url_match program_html +webarena.646,False,reddit,url_match program_html +webarena.647,False,reddit,url_match program_html +webarena.648,False,reddit,url_match program_html +webarena.649,False,reddit,url_match program_html +webarena.650,False,reddit,url_match program_html +webarena.651,False,reddit,url_match program_html +webarena.652,False,reddit,url_match program_html +webarena.653,False,shopping,url_match program_html +webarena.654,False,shopping,url_match program_html +webarena.655,False,shopping,url_match program_html +webarena.656,False,shopping,url_match program_html +webarena.657,False,shopping,url_match program_html +webarena.658,False,gitlab,url_match program_html +webarena.659,False,gitlab,url_match program_html +webarena.660,False,gitlab,url_match program_html +webarena.661,False,gitlab,url_match program_html +webarena.662,False,gitlab,url_match program_html +webarena.663,False,gitlab,url_match program_html +webarena.664,False,gitlab,url_match program_html +webarena.665,False,gitlab,url_match program_html +webarena.666,False,gitlab,url_match program_html +webarena.667,False,gitlab,url_match program_html +webarena.668,False,gitlab,url_match program_html +webarena.669,False,gitlab,url_match program_html +webarena.670,False,gitlab,url_match program_html +webarena.671,False,shopping reddit,url_match program_html +webarena.672,False,shopping reddit,url_match program_html +webarena.673,False,shopping reddit,url_match program_html +webarena.674,False,shopping reddit,url_match program_html +webarena.675,False,shopping reddit,url_match program_html +webarena.676,False,shopping_admin,url_match program_html +webarena.677,False,shopping_admin,url_match program_html +webarena.678,False,shopping_admin,url_match program_html +webarena.679,False,shopping_admin,url_match program_html +webarena.680,False,shopping_admin,url_match program_html +webarena.681,False,reddit gitlab,url_match program_html +webarena.682,False,reddit gitlab,url_match program_html +webarena.683,False,reddit gitlab,url_match program_html +webarena.684,False,reddit gitlab,url_match program_html +webarena.685,False,reddit gitlab,url_match program_html +webarena.686,False,reddit gitlab,url_match program_html +webarena.687,False,reddit gitlab,url_match program_html +webarena.688,False,reddit gitlab,url_match program_html +webarena.689,False,shopping,url_match program_html +webarena.690,False,shopping,url_match program_html +webarena.691,False,shopping,url_match program_html +webarena.692,False,shopping,url_match program_html +webarena.693,False,shopping,url_match program_html +webarena.694,False,shopping_admin,url_match program_html +webarena.695,False,shopping_admin,url_match program_html +webarena.696,False,shopping_admin,url_match program_html +webarena.697,False,shopping_admin,url_match program_html +webarena.698,False,shopping_admin,url_match program_html +webarena.699,False,shopping_admin,url_match program_html +webarena.700,False,shopping_admin,url_match program_html +webarena.701,False,shopping_admin,url_match program_html +webarena.702,False,shopping_admin,url_match program_html +webarena.703,False,shopping_admin,url_match program_html +webarena.704,False,shopping_admin,url_match program_html +webarena.705,False,shopping_admin,url_match program_html +webarena.706,False,shopping_admin,url_match program_html +webarena.707,False,shopping_admin,url_match program_html +webarena.708,False,shopping_admin,url_match program_html +webarena.709,False,shopping_admin,url_match program_html +webarena.710,False,shopping_admin,url_match program_html +webarena.711,False,shopping_admin,url_match program_html +webarena.712,False,shopping_admin,url_match program_html +webarena.713,False,shopping_admin,url_match program_html +webarena.714,False,reddit,program_html +webarena.715,False,reddit,program_html +webarena.716,False,reddit,program_html +webarena.717,False,reddit,program_html +webarena.718,False,reddit,program_html +webarena.719,False,reddit,program_html +webarena.720,False,reddit,program_html +webarena.721,False,reddit,program_html +webarena.722,False,reddit,program_html +webarena.723,False,reddit,string_match +webarena.724,False,reddit,program_html +webarena.725,False,reddit,program_html +webarena.726,False,reddit,string_match +webarena.727,False,reddit,program_html +webarena.728,False,reddit,program_html +webarena.729,False,reddit,program_html +webarena.730,False,reddit,program_html +webarena.731,False,reddit,program_html +webarena.732,False,reddit,program_html +webarena.733,False,reddit,program_html +webarena.734,False,reddit,program_html +webarena.735,False,reddit,program_html +webarena.736,False,gitlab,program_html +webarena.737,False,wikipedia map,program_html +webarena.738,False,wikipedia map,program_html +webarena.739,False,wikipedia map,program_html +webarena.740,False,wikipedia map,program_html +webarena.741,False,wikipedia map,program_html +webarena.742,False,gitlab,program_html +webarena.743,False,gitlab,program_html +webarena.744,False,gitlab,program_html +webarena.745,False,gitlab,program_html +webarena.746,False,gitlab,program_html +webarena.747,False,gitlab,program_html +webarena.748,False,gitlab,program_html +webarena.749,False,gitlab,program_html +webarena.750,False,gitlab,program_html +webarena.751,False,gitlab,program_html +webarena.752,False,gitlab,program_html +webarena.753,False,gitlab,program_html +webarena.754,False,gitlab,program_html +webarena.755,False,gitlab,program_html +webarena.756,False,gitlab,program_html +webarena.757,False,map,program_html +webarena.758,False,map,program_html +webarena.759,False,map shopping_admin,program_html +webarena.760,False,map shopping_admin,program_html +webarena.761,False,map,program_html +webarena.762,False,map,program_html +webarena.763,False,map,program_html +webarena.764,False,map,program_html +webarena.765,False,map,program_html +webarena.766,False,map,program_html +webarena.767,False,map,program_html +webarena.768,False,shopping_admin,program_html +webarena.769,False,shopping_admin,program_html +webarena.770,False,shopping_admin,program_html +webarena.771,False,shopping_admin,program_html +webarena.772,False,shopping_admin,program_html +webarena.773,False,shopping_admin,program_html +webarena.774,False,shopping_admin,program_html +webarena.775,False,shopping_admin,program_html +webarena.776,False,shopping_admin,program_html +webarena.777,False,shopping_admin,program_html +webarena.778,False,shopping_admin,program_html +webarena.779,False,shopping_admin,program_html +webarena.780,False,shopping_admin,program_html +webarena.781,False,shopping_admin,program_html +webarena.782,False,shopping_admin,program_html +webarena.783,False,gitlab,string_match +webarena.784,False,gitlab,string_match +webarena.785,False,gitlab,string_match +webarena.786,False,gitlab,string_match +webarena.787,False,gitlab,string_match +webarena.788,False,gitlab,string_match +webarena.789,False,gitlab,string_match +webarena.790,False,shopping_admin,string_match +webarena.791,False,gitlab reddit,string_match +webarena.792,False,shopping,string_match +webarena.793,False,shopping,string_match +webarena.794,False,shopping,string_match +webarena.795,False,shopping,string_match +webarena.796,False,shopping,string_match +webarena.797,False,shopping,string_match +webarena.798,False,shopping,string_match +webarena.799,False,gitlab,program_html +webarena.800,False,gitlab,program_html +webarena.801,False,gitlab,program_html +webarena.802,False,gitlab,program_html +webarena.803,False,gitlab,program_html +webarena.804,False,gitlab,program_html +webarena.805,False,gitlab,url_match program_html +webarena.806,False,gitlab,url_match program_html +webarena.807,False,gitlab,url_match program_html +webarena.808,False,gitlab,url_match program_html +webarena.809,False,gitlab,url_match program_html +webarena.810,False,gitlab,program_html +webarena.811,False,gitlab,program_html diff --git a/browsergym/experiments/src/browsergym/experiments/task_metadata/workarena.csv b/browsergym/experiments/src/browsergym/experiments/task_metadata/workarena.csv new file mode 100644 index 00000000..3d20a522 --- /dev/null +++ b/browsergym/experiments/src/browsergym/experiments/task_metadata/workarena.csv @@ -0,0 +1,716 @@ +task_name,level,category +workarena.servicenow.multi-chart-value-retrieval,l1,dashboard +workarena.servicenow.multi-chart-min-max-retrieval,l1,dashboard +workarena.servicenow.single-chart-value-retrieval,l1,dashboard +workarena.servicenow.single-chart-min-max-retrieval,l1,dashboard +workarena.servicenow.create-change-request,l1,form +workarena.servicenow.create-incident,l1,form +workarena.servicenow.create-hardware-asset,l1,form +workarena.servicenow.create-problem,l1,form +workarena.servicenow.create-user,l1,form +workarena.servicenow.knowledge-base-search,l1,knowledge +workarena.servicenow.filter-asset-list,l1,list-filter +workarena.servicenow.filter-change-request-list,l1,list-filter +workarena.servicenow.filter-hardware-list,l1,list-filter +workarena.servicenow.filter-incident-list,l1,list-filter +workarena.servicenow.filter-service-catalog-item-list,l1,list-filter +workarena.servicenow.filter-user-list,l1,list-filter +workarena.servicenow.sort-asset-list,l1,list-sort +workarena.servicenow.sort-change-request-list,l1,list-sort +workarena.servicenow.sort-hardware-list,l1,list-sort +workarena.servicenow.sort-incident-list,l1,list-sort +workarena.servicenow.sort-service-catalog-item-list,l1,list-sort +workarena.servicenow.sort-user-list,l1,list-sort +workarena.servicenow.all-menu,l1,menu +workarena.servicenow.impersonation,l1,menu +workarena.servicenow.order-developer-laptop,l1,service catalog +workarena.servicenow.order-ipad-mini,l1,service catalog +workarena.servicenow.order-ipad-pro,l1,service catalog +workarena.servicenow.order-sales-laptop,l1,service catalog +workarena.servicenow.order-standard-laptop,l1,service catalog +workarena.servicenow.order-apple-watch,l1,service catalog +workarena.servicenow.order-apple-mac-book-pro15,l1,service catalog +workarena.servicenow.order-development-laptop-p-c,l1,service catalog +workarena.servicenow.order-loaner-laptop,l1,service catalog +workarena.servicenow.basic-filter-problems-and-mark-duplicates-small-l2,l2,planning_and_problem_solving +workarena.servicenow.basic-filter-problems-and-mark-duplicates-medium-l2,l2,planning_and_problem_solving +workarena.servicenow.basic-filter-problems-and-mark-duplicates-large-l2,l2,planning_and_problem_solving +workarena.servicenow.priority-filter-problems-and-mark-duplicates-small-l2,l2,planning_and_problem_solving +workarena.servicenow.priority-filter-problems-and-mark-duplicates-medium-l2,l2,planning_and_problem_solving +workarena.servicenow.priority-filter-problems-and-mark-duplicates-large-l2,l2,planning_and_problem_solving +workarena.servicenow.high-priority-filter-problems-and-mark-duplicates-small-l2,l2,planning_and_problem_solving +workarena.servicenow.high-priority-filter-problems-and-mark-duplicates-medium-l2,l2,planning_and_problem_solving +workarena.servicenow.high-priority-filter-problems-and-mark-duplicates-large-l2,l2,planning_and_problem_solving +workarena.servicenow.workload-balancing-small-l2,l2,planning_and_problem_solving +workarena.servicenow.workload-balancing-medium-l2,l2,planning_and_problem_solving +workarena.servicenow.workload-balancing-large-l2,l2,planning_and_problem_solving +workarena.servicenow.work-assignment-small-l2,l2,planning_and_problem_solving +workarena.servicenow.work-assignment-medium-l2,l2,planning_and_problem_solving +workarena.servicenow.work-assignment-large-l2,l2,planning_and_problem_solving +workarena.servicenow.priority-assignment-small-l2,l2,planning_and_problem_solving +workarena.servicenow.priority-assignment-medium-l2,l2,planning_and_problem_solving +workarena.servicenow.priority-assignment-large-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-basic-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-wide-basic-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-fix-basic-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-fix-wide-basic-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-basic-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-wide-basic-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-fix-basic-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-fix-wide-basic-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-basic-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-wide-basic-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-fix-basic-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-fix-wide-basic-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-basic-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-wide-basic-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-fix-basic-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-fix-wide-basic-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-priority-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-wide-priority-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-fix-priority-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-fix-wide-priority-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-wide-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-fix-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-fix-wide-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-tight-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-wide-schedule-tight-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-fix-tight-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-fix-wide-schedule-tight-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-tight-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-wide-tight-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-fix-tight-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.two-changes-fix-wide-tight-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-priority-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-wide-priority-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-fix-priority-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-fix-wide-priority-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-wide-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-fix-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-fix-wide-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-tight-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-wide-schedule-tight-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-fix-tight-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-fix-wide-schedule-tight-uniform-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-tight-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-wide-tight-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-fix-tight-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.three-changes-fix-wide-tight-priority-varied-risk-change-request-scheduling-l2,l2,planning_and_problem_solving +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-developer-laptop-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-orderi-pad-mini-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-orderi-pad-pro-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-sales-laptop-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-standard-laptop-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-apple-watch-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-apple-macbook-pro15-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-development-laptop-p-c-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-loaner-laptop-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-min-create-incident-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-min-create-problem-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-min-filter-asset-list-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-min-filter-hardware-list-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-min-filter-incident-list-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-min-filter-user-list-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-filter-asset-list-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-filter-hardware-list-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-filter-incident-list-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-filter-user-list-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-apple-watch-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-apple-watch2-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-apple-ipad3-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-apple-iphone13pro-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-apple-iphone13-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-galaxy-note20-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-google-nexus7-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-microsoft-surface-pro3-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-pixel4a-l2,l2,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-windows-surface-pro4-l2,l2,information_retrieval +workarena.servicenow.get-warranty-expiration-date-l2,l2,information_retrieval +workarena.servicenow.filter-requested-items-and-order-developer-laptop-l2,l2,information_retrieval +workarena.servicenow.filter-requested-items-and-order-ipad-mini-l2,l2,information_retrieval +workarena.servicenow.filter-requested-items-and-order-ipad-pro-l2,l2,information_retrieval +workarena.servicenow.filter-requested-items-and-order-sales-laptop-l2,l2,information_retrieval +workarena.servicenow.filter-requested-items-and-order-standard-laptop-l2,l2,information_retrieval +workarena.servicenow.filter-requested-items-and-order-apple-watch-l2,l2,information_retrieval +workarena.servicenow.filter-requested-items-and-order-apple-mac-book-pro15-l2,l2,information_retrieval +workarena.servicenow.filter-requested-items-and-order-development-laptop-p-c-l2,l2,information_retrieval +workarena.servicenow.filter-requested-items-and-order-loaner-laptop-l2,l2,information_retrieval +workarena.servicenow.basic-expense-management-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.date-based-expense-management-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.amount-based-expense-management-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.easy-expense-management-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.basic-expense-management-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.date-based-expense-management-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.amount-based-expense-management-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.easy-expense-management-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.basic-expense-management-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.date-based-expense-management-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.amount-based-expense-management-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.easy-expense-management-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-find-total-return-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-find-total-return-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-find-total-return-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-and-find-total-return-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-and-find-total-return-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-and-find-total-return-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-find-total-return-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-find-total-return-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-find-total-return-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-find-total-return-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-find-total-return-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-find-total-return-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-two-items-uniform-expenses-and-find-total-return-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-and-find-total-return-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-and-find-total-return-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-select-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-select-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-select-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-and-select-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-and-select-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-and-select-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-select-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-select-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-select-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-select-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-select-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-select-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-two-items-uniform-expenses-and-select-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-and-select-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-and-select-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-find-total-return-and-select-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-find-total-return-and-select-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-find-total-return-and-select-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-find-total-return-and-select-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-find-total-return-and-select-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-find-total-return-and-select-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-find-total-return-and-select-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-find-total-return-and-select-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-find-total-return-and-select-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-find-total-return-and-select-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-find-total-return-and-select-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-find-total-return-and-select-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-two-items-uniform-expenses-find-total-return-and-select-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-find-total-return-and-select-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-find-total-return-and-select-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-delete-wrong-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-delete-wrong-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-delete-wrong-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-delete-wrong-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-delete-wrong-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-delete-wrong-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-delete-wrong-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-delete-wrong-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-delete-wrong-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-two-items-uniform-expenses-and-delete-wrong-investments-small-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-and-delete-wrong-investments-medium-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-and-delete-wrong-investments-large-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-developer-laptop-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-orderi-pad-mini-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-orderi-pad-pro-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-sales-laptop-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-standard-laptop-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-apple-watch-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-apple-macbook-pro15-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-development-laptop-p-c-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-loaner-laptop-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-developer-laptop-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-orderi-pad-mini-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-orderi-pad-pro-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-sales-laptop-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-standard-laptop-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-apple-watch-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-apple-macbook-pro15-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-development-laptop-p-c-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-loaner-laptop-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-developer-laptop-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-orderi-pad-mini-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-orderi-pad-pro-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-sales-laptop-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-standard-laptop-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-apple-watch-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-apple-macbook-pro15-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-development-laptop-p-c-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-loaner-laptop-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-create-incident-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-create-incident-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-create-incident-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-create-problem-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-create-problem-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-create-problem-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-greater-filter-asset-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-greater-filter-hardware-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-greater-filter-incident-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-greater-filter-user-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-lesser-filter-asset-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-lesser-filter-hardware-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-lesser-filter-incident-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-lesser-filter-user-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-greater-filter-asset-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-lesser-filter-asset-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-greater-filter-hardware-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-lesser-filter-hardware-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-greater-filter-incident-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-lesser-filter-incident-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-greater-filter-user-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-lesser-filter-user-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-greater-filter-asset-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-lesser-filter-asset-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-greater-filter-hardware-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-lesser-filter-hardware-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-greater-filter-incident-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-lesser-filter-incident-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-greater-filter-user-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-lesser-filter-user-list-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-apple-watch-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-apple-watch2-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-apple-ipad3-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-apple-iphone13pro-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-apple-iphone13-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-galaxy-note20-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-google-nexus7-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-microsoft-surface-pro3-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-pixel4a-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-windows-surface-pro4-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-apple-watch-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-apple-watch2-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-apple-ipad3-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-apple-iphone13pro-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-apple-iphone13-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-galaxy-note20-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-google-nexus7-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-microsoft-surface-pro3-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-pixel4a-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-windows-surface-pro4-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-apple-watch-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-apple-watch2-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-apple-ipad3-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-apple-iphone13pro-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-apple-iphone13-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-galaxy-note20-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-google-nexus7-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-microsoft-surface-pro3-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-pixel4a-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-windows-surface-pro4-l2,l2,data_driven_decision_making_and_reasoning +workarena.servicenow.navigate-and-create-user-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-create-incident-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-create-change-request-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-create-problem-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-create-hardware-asset-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-order-standard-laptop-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-order-sales-laptop-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-order-developer-laptop-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-order-ipad-pro-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-order-ipad-mini-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-order-apple-watch-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-order-apple-mac-book-pro15-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-order-development-laptop-p-c-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-order-loaner-laptop-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-filter-asset-list-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-filter-user-list-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-filter-incident-list-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-filter-change-request-list-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-filter-hardware-list-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-filter-service-catalog-item-list-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-sort-asset-list-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-sort-user-list-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-sort-incident-list-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-sort-change-request-list-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-sort-hardware-list-l2,l2,sophisticated_memory +workarena.servicenow.navigate-and-sort-service-catalog-item-list-l2,l2,sophisticated_memory +workarena.servicenow.off-board-user-l2,l2,sophisticated_memory +workarena.servicenow.on-board-user-l2,l2,sophisticated_memory +workarena.servicenow.infeasible-navigate-and-create-user-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-incident-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-change-request-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-problem-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-hardware-asset-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-user-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-incident-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-change-request-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-problem-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-hardware-asset-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-standard-laptop-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-sales-laptop-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-developer-laptop-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-ipad-pro-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-ipad-mini-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-apple-watch-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-apple-mac-book-pro15-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-development-laptop-p-c-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-loaner-laptop-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-standard-laptop-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-sales-laptop-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-developer-laptop-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-ipad-pro-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-ipad-mini-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-apple-watch-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-apple-mac-book-pro15-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-development-laptop-p-c-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-loaner-laptop-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-asset-list-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-user-list-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-incident-list-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-change-request-list-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-hardware-list-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-service-catalog-item-list-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-asset-list-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-user-list-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-incident-list-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-change-request-list-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-hardware-list-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-service-catalog-item-list-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-asset-list-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-user-list-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-incident-list-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-change-request-list-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-hardware-list-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-service-catalog-item-list-with-reason-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-asset-list-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-user-list-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-incident-list-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-change-request-list-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-hardware-list-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-service-catalog-item-list-l2,l2,contextual_understanding_infeasible_tasks +workarena.servicenow.basic-filter-problems-and-mark-duplicates-small-l3,l3,planning_and_problem_solving +workarena.servicenow.basic-filter-problems-and-mark-duplicates-medium-l3,l3,planning_and_problem_solving +workarena.servicenow.basic-filter-problems-and-mark-duplicates-large-l3,l3,planning_and_problem_solving +workarena.servicenow.priority-filter-problems-and-mark-duplicates-small-l3,l3,planning_and_problem_solving +workarena.servicenow.priority-filter-problems-and-mark-duplicates-medium-l3,l3,planning_and_problem_solving +workarena.servicenow.priority-filter-problems-and-mark-duplicates-large-l3,l3,planning_and_problem_solving +workarena.servicenow.high-priority-filter-problems-and-mark-duplicates-small-l3,l3,planning_and_problem_solving +workarena.servicenow.high-priority-filter-problems-and-mark-duplicates-medium-l3,l3,planning_and_problem_solving +workarena.servicenow.high-priority-filter-problems-and-mark-duplicates-large-l3,l3,planning_and_problem_solving +workarena.servicenow.workload-balancing-small-l3,l3,planning_and_problem_solving +workarena.servicenow.workload-balancing-medium-l3,l3,planning_and_problem_solving +workarena.servicenow.workload-balancing-large-l3,l3,planning_and_problem_solving +workarena.servicenow.work-assignment-small-l3,l3,planning_and_problem_solving +workarena.servicenow.work-assignment-medium-l3,l3,planning_and_problem_solving +workarena.servicenow.work-assignment-large-l3,l3,planning_and_problem_solving +workarena.servicenow.priority-assignment-small-l3,l3,planning_and_problem_solving +workarena.servicenow.priority-assignment-medium-l3,l3,planning_and_problem_solving +workarena.servicenow.priority-assignment-large-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-basic-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-wide-basic-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-fix-basic-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-fix-wide-basic-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-basic-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-wide-basic-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-fix-basic-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-fix-wide-basic-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-basic-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-wide-basic-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-fix-basic-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-fix-wide-basic-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-basic-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-wide-basic-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-fix-basic-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-fix-wide-basic-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-priority-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-wide-priority-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-fix-priority-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-fix-wide-priority-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-wide-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-fix-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-fix-wide-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-tight-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-wide-schedule-tight-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-fix-tight-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-fix-wide-schedule-tight-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-tight-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-wide-tight-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-fix-tight-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.two-changes-fix-wide-tight-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-priority-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-wide-priority-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-fix-priority-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-fix-wide-priority-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-wide-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-fix-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-fix-wide-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-tight-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-wide-schedule-tight-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-fix-tight-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-fix-wide-schedule-tight-uniform-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-tight-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-wide-tight-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-fix-tight-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.three-changes-fix-wide-tight-priority-varied-risk-change-request-scheduling-l3,l3,planning_and_problem_solving +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-developer-laptop-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-orderi-pad-mini-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-orderi-pad-pro-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-sales-laptop-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-standard-laptop-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-apple-watch-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-apple-macbook-pro15-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-development-laptop-p-c-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-catalog-and-max-order-loaner-laptop-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-min-create-incident-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-min-create-problem-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-min-filter-asset-list-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-min-filter-hardware-list-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-min-filter-incident-list-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-min-filter-user-list-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-filter-asset-list-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-filter-hardware-list-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-filter-incident-list-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-filter-user-list-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-apple-watch-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-apple-watch2-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-apple-ipad3-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-apple-iphone13pro-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-apple-iphone13-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-galaxy-note20-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-google-nexus7-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-microsoft-surface-pro3-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-pixel4a-l3,l3,information_retrieval +workarena.servicenow.dashboard-retrieve-incident-and-max-request-windows-surface-pro4-l3,l3,information_retrieval +workarena.servicenow.get-warranty-expiration-date-l3,l3,information_retrieval +workarena.servicenow.filter-requested-items-and-order-developer-laptop-l3,l3,information_retrieval +workarena.servicenow.filter-requested-items-and-order-ipad-mini-l3,l3,information_retrieval +workarena.servicenow.filter-requested-items-and-order-ipad-pro-l3,l3,information_retrieval +workarena.servicenow.filter-requested-items-and-order-sales-laptop-l3,l3,information_retrieval +workarena.servicenow.filter-requested-items-and-order-standard-laptop-l3,l3,information_retrieval +workarena.servicenow.filter-requested-items-and-order-apple-watch-l3,l3,information_retrieval +workarena.servicenow.filter-requested-items-and-order-apple-mac-book-pro15-l3,l3,information_retrieval +workarena.servicenow.filter-requested-items-and-order-development-laptop-p-c-l3,l3,information_retrieval +workarena.servicenow.filter-requested-items-and-order-loaner-laptop-l3,l3,information_retrieval +workarena.servicenow.basic-expense-management-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.date-based-expense-management-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.amount-based-expense-management-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.easy-expense-management-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.basic-expense-management-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.date-based-expense-management-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.amount-based-expense-management-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.easy-expense-management-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.basic-expense-management-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.date-based-expense-management-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.amount-based-expense-management-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.easy-expense-management-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-find-total-return-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-find-total-return-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-find-total-return-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-and-find-total-return-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-and-find-total-return-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-and-find-total-return-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-find-total-return-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-find-total-return-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-find-total-return-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-find-total-return-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-find-total-return-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-find-total-return-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-two-items-uniform-expenses-and-find-total-return-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-and-find-total-return-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-and-find-total-return-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-select-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-select-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-select-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-and-select-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-and-select-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-and-select-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-select-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-select-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-select-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-select-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-select-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-select-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-two-items-uniform-expenses-and-select-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-and-select-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-and-select-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-find-total-return-and-select-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-find-total-return-and-select-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-find-total-return-and-select-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-find-total-return-and-select-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-find-total-return-and-select-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-trivial-expenses-find-total-return-and-select-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-find-total-return-and-select-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-find-total-return-and-select-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-find-total-return-and-select-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-find-total-return-and-select-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-find-total-return-and-select-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-find-total-return-and-select-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-two-items-uniform-expenses-find-total-return-and-select-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-find-total-return-and-select-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-find-total-return-and-select-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-delete-wrong-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-delete-wrong-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-random-expenses-and-delete-wrong-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-delete-wrong-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-delete-wrong-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-expenses-and-delete-wrong-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-delete-wrong-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-delete-wrong-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-single-item-uniform-expenses-and-delete-wrong-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-two-items-uniform-expenses-and-delete-wrong-investments-small-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-and-delete-wrong-investments-medium-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.filter-three-items-uniform-expenses-and-delete-wrong-investments-large-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-developer-laptop-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-orderi-pad-mini-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-orderi-pad-pro-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-sales-laptop-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-standard-laptop-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-apple-watch-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-apple-macbook-pro15-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-development-laptop-p-c-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mean-order-loaner-laptop-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-developer-laptop-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-orderi-pad-mini-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-orderi-pad-pro-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-sales-laptop-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-standard-laptop-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-apple-watch-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-apple-macbook-pro15-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-development-laptop-p-c-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-median-order-loaner-laptop-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-developer-laptop-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-orderi-pad-mini-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-orderi-pad-pro-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-sales-laptop-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-standard-laptop-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-apple-watch-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-apple-macbook-pro15-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-development-laptop-p-c-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-catalog-and-mode-order-loaner-laptop-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-create-incident-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-create-incident-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-create-incident-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-create-problem-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-create-problem-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-create-problem-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-greater-filter-asset-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-greater-filter-hardware-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-greater-filter-incident-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-greater-filter-user-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-lesser-filter-asset-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-lesser-filter-hardware-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-lesser-filter-incident-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-lesser-filter-user-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-greater-filter-asset-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-lesser-filter-asset-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-greater-filter-hardware-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-lesser-filter-hardware-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-greater-filter-incident-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-lesser-filter-incident-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-greater-filter-user-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-lesser-filter-user-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-greater-filter-asset-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-lesser-filter-asset-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-greater-filter-hardware-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-lesser-filter-hardware-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-greater-filter-incident-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-lesser-filter-incident-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-greater-filter-user-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-lesser-filter-user-list-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-apple-watch-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-apple-watch2-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-apple-ipad3-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-apple-iphone13pro-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-apple-iphone13-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-galaxy-note20-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-google-nexus7-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-microsoft-surface-pro3-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-pixel4a-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mean-request-windows-surface-pro4-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-apple-watch-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-apple-watch2-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-apple-ipad3-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-apple-iphone13pro-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-apple-iphone13-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-galaxy-note20-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-google-nexus7-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-microsoft-surface-pro3-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-pixel4a-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-median-request-windows-surface-pro4-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-apple-watch-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-apple-watch2-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-apple-ipad3-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-apple-iphone13pro-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-apple-iphone13-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-galaxy-note20-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-google-nexus7-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-microsoft-surface-pro3-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-pixel4a-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.dashboard-retrieve-incident-and-mode-request-windows-surface-pro4-l3,l3,data_driven_decision_making_and_reasoning +workarena.servicenow.navigate-and-create-user-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-create-incident-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-create-change-request-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-create-problem-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-create-hardware-asset-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-order-standard-laptop-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-order-sales-laptop-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-order-developer-laptop-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-order-ipad-pro-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-order-ipad-mini-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-order-apple-watch-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-order-apple-mac-book-pro15-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-order-development-laptop-p-c-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-order-loaner-laptop-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-filter-asset-list-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-filter-user-list-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-filter-incident-list-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-filter-change-request-list-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-filter-hardware-list-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-filter-service-catalog-item-list-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-sort-asset-list-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-sort-user-list-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-sort-incident-list-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-sort-change-request-list-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-sort-hardware-list-l3,l3,sophisticated_memory +workarena.servicenow.navigate-and-sort-service-catalog-item-list-l3,l3,sophisticated_memory +workarena.servicenow.off-board-user-l3,l3,sophisticated_memory +workarena.servicenow.on-board-user-l3,l3,sophisticated_memory +workarena.servicenow.infeasible-navigate-and-create-user-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-incident-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-change-request-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-problem-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-hardware-asset-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-user-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-incident-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-change-request-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-problem-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-create-hardware-asset-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-standard-laptop-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-sales-laptop-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-developer-laptop-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-ipad-pro-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-ipad-mini-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-apple-watch-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-apple-mac-book-pro15-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-development-laptop-p-c-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-loaner-laptop-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-standard-laptop-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-sales-laptop-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-developer-laptop-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-ipad-pro-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-ipad-mini-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-apple-watch-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-apple-mac-book-pro15-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-development-laptop-p-c-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-order-loaner-laptop-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-asset-list-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-user-list-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-incident-list-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-change-request-list-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-hardware-list-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-service-catalog-item-list-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-asset-list-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-user-list-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-incident-list-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-change-request-list-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-hardware-list-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-filter-service-catalog-item-list-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-asset-list-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-user-list-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-incident-list-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-change-request-list-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-hardware-list-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-service-catalog-item-list-with-reason-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-asset-list-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-user-list-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-incident-list-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-change-request-list-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-hardware-list-l3,l3,contextual_understanding_infeasible_tasks +workarena.servicenow.infeasible-navigate-and-sort-service-catalog-item-list-l3,l3,contextual_understanding_infeasible_tasks diff --git a/browsergym/pyproject.toml b/browsergym/pyproject.toml index fab70d4f..9d296476 100644 --- a/browsergym/pyproject.toml +++ b/browsergym/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "browsergym-webarena==0.8.1", "browsergym-visualwebarena==0.8.1", "browsergym-experiments==0.8.1", - "browsergym-workarena", + "browsergym-workarena>=0.4.1", ] [tool.setuptools] diff --git a/browsergym/visualwebarena/src/browsergym/visualwebarena/config.py b/browsergym/visualwebarena/src/browsergym/visualwebarena/config.py index 3cc2f5dd..2077e7e0 100644 --- a/browsergym/visualwebarena/src/browsergym/visualwebarena/config.py +++ b/browsergym/visualwebarena/src/browsergym/visualwebarena/config.py @@ -1,9 +1,11 @@ TASK_IDS = range(910) # import visualwebarena +# import importlib.resources +# import json # all_configs_str = importlib.resources.files(visualwebarena).joinpath("test_raw.json").read_text() # all_configs = json.loads(all_configs_str) -# task_ids_with_reset = [task["task_id"] for task in x if task["require_reset"] == True] +# task_ids_with_reset = [task["task_id"] for task in all_configs if task["require_reset"] == True] TASK_IDS_WITH_RESET = [ 4, 5, diff --git a/dev/requirements.txt b/dev/requirements.txt index 2de76f76..dcb5c57e 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -10,3 +10,4 @@ tenacity -e ../browsergym/webarena # local package -e ../browsergym/visualwebarena # local package -e ../browsergym/experiments # local package +browsergym-workarena diff --git a/tests/experiments/test_benchmark.py b/tests/experiments/test_benchmark.py new file mode 100644 index 00000000..117dca28 --- /dev/null +++ b/tests/experiments/test_benchmark.py @@ -0,0 +1,148 @@ +import dataclasses +import re +import tempfile + +import numpy as np + +from browsergym.core.action.base import AbstractActionSet +from browsergym.experiments.agent import Agent +from browsergym.experiments.benchmark import ( + BENCHMARKS, + Benchmark, + HighLevelActionSetArgs, + _make_env_args_list_from_repeat_tasks, +) +from browsergym.experiments.loop import ( + AbstractAgentArgs, + EnvArgs, + ExpArgs, + get_exp_result, +) +from browsergym.utils.obs import flatten_axtree_to_str + + +class MiniwobTestAgent(Agent): + + def __init__(self, action_set: AbstractActionSet): + self.action_set = action_set + + def obs_preprocessor(self, obs: dict): + return {"axtree_txt": flatten_axtree_to_str(obs["axtree_object"])} + + def get_action(self, obs: dict) -> tuple[str, dict]: + match = re.search(r"^\s*\[(\d+)\].*button", obs["axtree_txt"], re.MULTILINE | re.IGNORECASE) + + if match: + bid = match.group(1) + action = f'click("{bid}")' + else: + raise Exception("Can't find the button's bid") + + return action, dict(think="I'm clicking the button as requested.") + + +@dataclasses.dataclass +class MiniwobTestAgentArgs(AbstractAgentArgs): + high_level_action_set: HighLevelActionSetArgs = None + + def make_agent(self): + return MiniwobTestAgent(action_set=self.high_level_action_set.make_action_set()) + + +def test_build_benchmarks(): + expected_bench_size = { + "miniwob_all": 125 * 10, + "miniwob_webgum": 56 * 10, + "miniwob_tiny_test": 2 * 2, + "miniwob_train": 107 * 10, + "miniwob_test": 18 * 10, + "webarena": 812, + "visualwebarena": 910, + "workarena_l1": 33 * 10, + "workarena_l1_sort": 6 * 10, + "workarena_l2_agent_curriculum_eval": 235, + "workarena_l3_agent_curriculum_eval": 235, + } + for name, benchmark_builder in BENCHMARKS.items(): + benchmark = benchmark_builder() + assert name == benchmark.name + assert benchmark.env_args_list # non-empty + assert benchmark.task_metadata is not None + assert len(benchmark.env_args_list) == expected_bench_size[name] + benchmark_bis = Benchmark.from_json(benchmark.to_json()) + assert benchmark.to_dict() == benchmark_bis.to_dict() + + +def test_benchmark_subset(): + benchmark: Benchmark = BENCHMARKS["miniwob_all"]() + + benchmark_subset = benchmark.subset_from_regexp(column="task_name", regexp="click") + assert len(benchmark_subset.env_args_list) == 31 * 10 + assert benchmark_subset.name == "miniwob_all[task_name=/click/]" + + benchmark_subset_1 = benchmark_subset.subset_from_regexp( + column="miniwob_category", regexp="original" + ) + benchmark_subset_2 = benchmark_subset.subset_from_glob( + column="miniwob_category", glob="original" + ) + + assert benchmark_subset_1.name == "miniwob_all[task_name=/click/][miniwob_category=/original/]" + assert benchmark_subset_2.name == "miniwob_all[task_name=/click/][miniwob_category=original]" + + dict_1 = benchmark_subset_1.to_dict() + dict_1.pop("name") + dict_2 = benchmark_subset_2.to_dict() + dict_2.pop("name") + + assert dict_1 == dict_2 + + +def test_run_mock_benchmark(): + benchmark = Benchmark( + name="miniwob_click_test", + high_level_action_set_args=HighLevelActionSetArgs( + subsets=["bid"], + multiaction=False, + strict=False, + retry_with_force=False, + demo_mode="off", + ), + env_args_list=_make_env_args_list_from_repeat_tasks( + task_list=["miniwob.click-test"], + max_steps=5, + n_repeats=2, + seeds_rng=np.random.RandomState(42), + ), + ) + + for env_args in benchmark.env_args_list: + agent_args = MiniwobTestAgentArgs( + high_level_action_set=benchmark.high_level_action_set_args + ) + exp_args = ExpArgs( + agent_args=agent_args, + env_args=env_args, + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + exp_args.prepare(tmp_dir) + exp_args.run() + exp_result = get_exp_result(exp_args.exp_dir) + exp_record = exp_result.get_exp_record() + + target = { + "env_args.task_name": "miniwob.click-test", + "env_args.headless": True, + "env_args.record_video": False, + "n_steps": 1, + "cum_reward": 1.0, + "terminated": True, + "truncated": False, + } + + assert len(exp_result.steps_info) == 2 + + for key, target_val in target.items(): + assert key in exp_record + assert exp_record[key] == target_val