diff --git a/.github/workflows/ui-tests-playwright.yml b/.github/workflows/ui-tests-playwright.yml index 4ae8484b8..cc2c56106 100644 --- a/.github/workflows/ui-tests-playwright.yml +++ b/.github/workflows/ui-tests-playwright.yml @@ -18,11 +18,11 @@ jobs: working-directory: ./frontend steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '20' cache: 'yarn' cache-dependency-path: frontend/yarn.lock - name: Install dependencies @@ -43,7 +43,7 @@ jobs: id: build-frontend - name: Run Playwright tests run: cd frontend && yarn playwright test - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 if: always() with: name: playwright-report diff --git a/backend/btrixcloud/basecrawls.py b/backend/btrixcloud/basecrawls.py index 2df1bfabb..67df51be5 100644 --- a/backend/btrixcloud/basecrawls.py +++ b/backend/btrixcloud/basecrawls.py @@ -31,7 +31,7 @@ CrawlSearchValuesResponse, ) from .pagination import paginated_format, DEFAULT_PAGE_SIZE -from .utils import dt_now +from .utils import dt_now, date_to_str if TYPE_CHECKING: from .crawlconfigs import CrawlConfigOps @@ -494,7 +494,7 @@ async def resolve_signed_urls( expire_at_str = "" if file_.expireAt: - expire_at_str = file_.expireAt.isoformat() + expire_at_str = date_to_str(file_.expireAt) out_files.append( CrawlFileOut( diff --git a/backend/btrixcloud/crawlmanager.py b/backend/btrixcloud/crawlmanager.py index 6bb3d1887..dd2ef3d11 100644 --- a/backend/btrixcloud/crawlmanager.py +++ b/backend/btrixcloud/crawlmanager.py @@ -9,7 +9,7 @@ from fastapi import HTTPException -from .utils import dt_now, to_k8s_date +from .utils import dt_now, date_to_str from .k8sapi import K8sAPI from .models import StorageRef, CrawlConfig, BgJobType @@ -53,7 +53,7 @@ async def run_profile_browser( "idle_timeout": os.environ.get("IDLE_TIMEOUT", "60"), "url": url, "vnc_password": secrets.token_hex(16), - "expire_time": to_k8s_date(dt_now() + timedelta(seconds=30)), + "expire_time": date_to_str(dt_now() + timedelta(seconds=30)), "crawler_image": crawler_image, } @@ -237,12 +237,12 @@ async def ping_profile_browser(self, browserid: str) -> None: """return ping profile browser""" expire_at = dt_now() + timedelta(seconds=30) await self._patch_job( - browserid, {"expireTime": to_k8s_date(expire_at)}, "profilejobs" + browserid, {"expireTime": date_to_str(expire_at)}, "profilejobs" ) async def rollover_restart_crawl(self, crawl_id: str) -> dict: """Rolling restart of crawl by updating restartTime field""" - update = to_k8s_date(dt_now()) + update = date_to_str(dt_now()) return await self._patch_job(crawl_id, {"restartTime": update}) async def scale_crawl(self, crawl_id: str, scale: int = 1) -> dict: diff --git a/backend/btrixcloud/crawls.py b/backend/btrixcloud/crawls.py index be1575313..1b9c6e801 100644 --- a/backend/btrixcloud/crawls.py +++ b/backend/btrixcloud/crawls.py @@ -19,7 +19,12 @@ import pymongo from .pagination import DEFAULT_PAGE_SIZE, paginated_format -from .utils import dt_now, parse_jsonl_error_messages, stream_dict_list_as_csv +from .utils import ( + dt_now, + date_to_str, + parse_jsonl_error_messages, + stream_dict_list_as_csv, +) from .basecrawls import BaseCrawlOps from .crawlmanager import CrawlManager from .models import ( @@ -714,8 +719,8 @@ async def get_crawl_stats( data["userid"] = str(crawl.userid) data["user"] = user_emails.get(crawl.userid) - data["started"] = str(crawl.started) - data["finished"] = str(crawl.finished) + data["started"] = date_to_str(crawl.started) if crawl.started else "" + data["finished"] = date_to_str(crawl.finished) if crawl.finished else "" data["duration"] = 0 duration_seconds = 0 diff --git a/backend/btrixcloud/db.py b/backend/btrixcloud/db.py index 50c1ead42..91b2bc705 100644 --- a/backend/btrixcloud/db.py +++ b/backend/btrixcloud/db.py @@ -43,6 +43,7 @@ def init_db(): client = motor.motor_asyncio.AsyncIOMotorClient( db_url, + tz_aware=True, uuidRepresentation="standard", connectTimeoutMS=120000, serverSelectionTimeoutMS=120000, diff --git a/backend/btrixcloud/operator/bgjobs.py b/backend/btrixcloud/operator/bgjobs.py index 2552ebaf6..59009d01d 100644 --- a/backend/btrixcloud/operator/bgjobs.py +++ b/backend/btrixcloud/operator/bgjobs.py @@ -4,7 +4,7 @@ import traceback from btrixcloud.utils import ( - from_k8s_date, + str_to_date, dt_now, ) @@ -45,7 +45,7 @@ async def finalize_background_job(self, data: MCDecoratorSyncData) -> dict: finished = None if completion_time: - finished = from_k8s_date(completion_time) + finished = str_to_date(completion_time) if not finished: finished = dt_now() diff --git a/backend/btrixcloud/operator/crawls.py b/backend/btrixcloud/operator/crawls.py index 561b58531..b974fe505 100644 --- a/backend/btrixcloud/operator/crawls.py +++ b/backend/btrixcloud/operator/crawls.py @@ -32,7 +32,7 @@ Organization, ) -from btrixcloud.utils import from_k8s_date, to_k8s_date, dt_now +from btrixcloud.utils import str_to_date, date_to_str, dt_now from .baseoperator import BaseOperator, Redis from .models import ( @@ -412,8 +412,8 @@ def _qa_configmap_update_needed(self, name, configmap): now = dt_now() resources = json.loads(configmap["data"]["qa-config.json"])["resources"] for resource in resources: - expire_at = datetime.fromisoformat(resource["expireAt"]) - if expire_at <= now: + expire_at = str_to_date(resource["expireAt"]) + if expire_at and expire_at <= now: print(f"Refreshing QA configmap for QA run: {name}") return True @@ -551,7 +551,7 @@ async def set_state( if actual_state: status.state = actual_state if finished: - status.finished = to_k8s_date(finished) + status.finished = date_to_str(finished) if actual_state != state: print( @@ -725,7 +725,7 @@ async def finalize_response( # keep parent until ttl expired, if any if status.finished: ttl = spec.get("ttlSecondsAfterFinished", DEFAULT_TTL) - finished = from_k8s_date(status.finished) + finished = str_to_date(status.finished) if finished and (dt_now() - finished).total_seconds() > ttl >= 0: print("CrawlJob expired, deleting: " + crawl.id) finalized = True @@ -802,7 +802,7 @@ async def sync_crawl_state( # but not right away in case crawler pod is just restarting. # avoids keeping redis pods around while no crawler pods are up # (eg. due to resource constraints) - last_active_time = from_k8s_date(status.lastActiveTime) + last_active_time = str_to_date(status.lastActiveTime) if last_active_time and ( (dt_now() - last_active_time).total_seconds() > REDIS_TTL ): @@ -820,7 +820,7 @@ async def sync_crawl_state( # update lastActiveTime if crawler is running if crawler_running: - status.lastActiveTime = to_k8s_date(dt_now()) + status.lastActiveTime = date_to_str(dt_now()) file_done = await redis.lpop(self.done_key) while file_done: @@ -969,7 +969,7 @@ async def increment_pod_exec_time( if status.state in WAITING_STATES: # reset lastUpdatedTime if at least 2 consecutive updates of non-running state if status.last_state in WAITING_STATES: - status.lastUpdatedTime = to_k8s_date(now) + status.lastUpdatedTime = date_to_str(now) return update_start_time = await self.crawl_ops.get_crawl_exec_last_update_time( @@ -995,7 +995,7 @@ async def increment_pod_exec_time( await self.crawl_ops.inc_crawl_exec_time( crawl.db_crawl_id, crawl.is_qa, 0, now ) - status.lastUpdatedTime = to_k8s_date(now) + status.lastUpdatedTime = date_to_str(now) return reason = None @@ -1041,7 +1041,7 @@ async def increment_pod_exec_time( if "running" in cstate: pod_state = "running" state = cstate["running"] - start_time = from_k8s_date(state.get("startedAt")) + start_time = str_to_date(state.get("startedAt")) if update_start_time and start_time and update_start_time > start_time: start_time = update_start_time @@ -1049,8 +1049,8 @@ async def increment_pod_exec_time( elif "terminated" in cstate: pod_state = "terminated" state = cstate["terminated"] - start_time = from_k8s_date(state.get("startedAt")) - end_time = from_k8s_date(state.get("finishedAt")) + start_time = str_to_date(state.get("startedAt")) + end_time = str_to_date(state.get("finishedAt")) if update_start_time and start_time and update_start_time > start_time: start_time = update_start_time @@ -1085,16 +1085,17 @@ async def increment_pod_exec_time( await self.crawl_ops.inc_crawl_exec_time( crawl.db_crawl_id, crawl.is_qa, exec_time, now ) - status.lastUpdatedTime = to_k8s_date(now) + status.lastUpdatedTime = date_to_str(now) - def should_mark_waiting(self, state, started): + def should_mark_waiting(self, state: TYPE_ALL_CRAWL_STATES, started: str) -> bool: """Should the crawl be marked as waiting for capacity?""" if state in RUNNING_STATES: return True if state == "starting": - started = from_k8s_date(started) - return (dt_now() - started).total_seconds() > STARTING_TIME_SECS + started_dt = str_to_date(started) + if started_dt: + return (dt_now() - started_dt).total_seconds() > STARTING_TIME_SECS return False @@ -1187,7 +1188,7 @@ async def log_crashes(self, crawl_id, pod_status: dict[str, PodInfo], redis): def get_log_line(self, message, details): """get crawler error line for logging""" err = { - "timestamp": dt_now().isoformat(), + "timestamp": date_to_str(dt_now()), "logLevel": "error", "context": "k8s", "message": message, @@ -1244,7 +1245,7 @@ async def is_crawl_stopping( # check timeout if timeout time exceeds elapsed time if crawl.timeout: elapsed = status.elapsedCrawlTime - last_updated_time = from_k8s_date(status.lastUpdatedTime) + last_updated_time = str_to_date(status.lastUpdatedTime) if last_updated_time: elapsed += int((dt_now() - last_updated_time).total_seconds()) @@ -1459,11 +1460,11 @@ async def mark_finished( ): print("already finished, ignoring mark_finished") if not status.finished: - status.finished = to_k8s_date(finished) + status.finished = date_to_str(finished) return False - status.finished = to_k8s_date(finished) + status.finished = date_to_str(finished) if state in SUCCESSFUL_STATES: await self.inc_crawl_complete_stats(crawl, finished) @@ -1524,7 +1525,7 @@ async def do_qa_run_finished_tasks( async def inc_crawl_complete_stats(self, crawl: CrawlSpec, finished: datetime): """Increment Crawl Stats""" - started = from_k8s_date(crawl.started) + started = str_to_date(crawl.started) if not started: print("Missing crawl start time, unable to increment crawl stats") return diff --git a/backend/btrixcloud/operator/cronjobs.py b/backend/btrixcloud/operator/cronjobs.py index 3aa3af6fa..74a43b0b4 100644 --- a/backend/btrixcloud/operator/cronjobs.py +++ b/backend/btrixcloud/operator/cronjobs.py @@ -4,7 +4,7 @@ from typing import Optional import yaml -from btrixcloud.utils import to_k8s_date, dt_now +from btrixcloud.utils import date_to_str, dt_now from .models import MCDecoratorSyncData, CJS, MCDecoratorSyncResponse from .baseoperator import BaseOperator @@ -31,7 +31,7 @@ def get_finished_response( """get final response to indicate cronjob created job is finished""" if not finished: - finished = to_k8s_date(dt_now()) + finished = date_to_str(dt_now()) status = None # set status on decorated job to indicate that its finished @@ -151,7 +151,7 @@ async def sync_cronjob_crawl( crawl_id, is_qa=False ) if finished: - finished_str = to_k8s_date(finished) + finished_str = date_to_str(finished) set_status = False # mark job as completed if not data.object["status"].get("succeeded"): diff --git a/backend/btrixcloud/operator/profiles.py b/backend/btrixcloud/operator/profiles.py index 03b6c5858..922b49d45 100644 --- a/backend/btrixcloud/operator/profiles.py +++ b/backend/btrixcloud/operator/profiles.py @@ -1,6 +1,6 @@ """ Operator handler for ProfileJobs """ -from btrixcloud.utils import from_k8s_date, dt_now +from btrixcloud.utils import str_to_date, dt_now from btrixcloud.models import StorageRef @@ -23,7 +23,7 @@ async def sync_profile_browsers(self, data: MCSyncData): """sync profile browsers""" spec = data.parent.get("spec", {}) - expire_time = from_k8s_date(spec.get("expireTime")) + expire_time = str_to_date(spec.get("expireTime")) browserid = spec.get("id") if expire_time and dt_now() >= expire_time: diff --git a/backend/btrixcloud/pages.py b/backend/btrixcloud/pages.py index 3d842bd64..a980567c4 100644 --- a/backend/btrixcloud/pages.py +++ b/backend/btrixcloud/pages.py @@ -31,7 +31,7 @@ PageNoteUpdatedResponse, ) from .pagination import DEFAULT_PAGE_SIZE, paginated_format -from .utils import from_k8s_date, str_list_to_bools, dt_now +from .utils import str_to_date, str_list_to_bools, dt_now if TYPE_CHECKING: from .crawls import CrawlOps @@ -112,7 +112,7 @@ def _get_page_from_dict( loadState=page_dict.get("loadState"), status=status, mime=page_dict.get("mime", "text/html"), - ts=(from_k8s_date(ts) if ts else dt_now()), + ts=(str_to_date(ts) if ts else dt_now()), ) p.compute_page_type() return p diff --git a/backend/btrixcloud/utils.py b/backend/btrixcloud/utils.py index 539b1e0e0..468c89242 100644 --- a/backend/btrixcloud/utils.py +++ b/backend/btrixcloud/utils.py @@ -33,7 +33,7 @@ def default(self, o: Any) -> str: return str(o) if isinstance(o, datetime): - return o.isoformat() + return date_to_str(o) return super().default(o) @@ -43,24 +43,19 @@ def get_templates_dir() -> str: return os.path.join(os.path.dirname(__file__), "templates") -def from_k8s_date(string: str) -> Optional[datetime]: +def str_to_date(string: str) -> Optional[datetime]: """convert k8s date string to datetime""" - return datetime.fromisoformat(string[:-1]) if string else None + return datetime.fromisoformat(string) if string else None -def to_k8s_date(dt_val: datetime) -> str: - """convert datetime to string for k8s""" - return dt_val.isoformat("T") + "Z" +def date_to_str(dt_val: datetime) -> str: + """convert date to isostring with "Z" """ + return dt_val.isoformat().replace("+00:00", "Z") def dt_now() -> datetime: """get current ts""" - return datetime.now(timezone.utc).replace(microsecond=0, tzinfo=None) - - -def ts_now() -> str: - """get current ts""" - return str(dt_now()) + return datetime.now(timezone.utc).replace(microsecond=0) def run_once_lock(name) -> bool: diff --git a/backend/test/test_collections.py b/backend/test/test_collections.py index 508beeb80..d0210e24d 100644 --- a/backend/test/test_collections.py +++ b/backend/test/test_collections.py @@ -134,6 +134,7 @@ def test_update_collection( global modified modified = data["modified"] assert modified + assert modified.endswith("Z") def test_rename_collection( diff --git a/backend/test/test_crawlconfigs.py b/backend/test/test_crawlconfigs.py index ec09e6a77..967f32db3 100644 --- a/backend/test/test_crawlconfigs.py +++ b/backend/test/test_crawlconfigs.py @@ -28,6 +28,14 @@ def test_crawl_config_usernames( assert data["modifiedByName"] assert data["lastStartedByName"] + created = data["created"] + assert created + assert created.endswith("Z") + + modified = data["modified"] + assert modified + assert modified.endswith("Z") + def test_add_crawl_config(crawler_auth_headers, default_org_id, sample_crawl_data): # Create crawl config diff --git a/backend/test/test_org_subs.py b/backend/test/test_org_subs.py index 880040be1..b48c5bb46 100644 --- a/backend/test/test_org_subs.py +++ b/backend/test/test_org_subs.py @@ -257,7 +257,7 @@ def test_update_sub(admin_auth_headers): "subId": "123", "status": "paused_payment_failed", "planId": "basic", - "futureCancelDate": "2028-12-26T01:02:03", + "futureCancelDate": "2028-12-26T01:02:03Z", "readOnlyOnCancel": False, } @@ -466,7 +466,7 @@ def test_subscription_events_log(admin_auth_headers, non_default_org_id): "oid": new_subs_oid, "status": "paused_payment_failed", "planId": "basic", - "futureCancelDate": "2028-12-26T01:02:03", + "futureCancelDate": "2028-12-26T01:02:03Z", "quotas": None, }, { @@ -534,7 +534,7 @@ def test_subscription_events_log_filter_sub_id(admin_auth_headers): "oid": new_subs_oid, "status": "paused_payment_failed", "planId": "basic", - "futureCancelDate": "2028-12-26T01:02:03", + "futureCancelDate": "2028-12-26T01:02:03Z", "quotas": None, }, { @@ -595,7 +595,7 @@ def test_subscription_events_log_filter_oid(admin_auth_headers): "oid": new_subs_oid, "status": "paused_payment_failed", "planId": "basic", - "futureCancelDate": "2028-12-26T01:02:03", + "futureCancelDate": "2028-12-26T01:02:03Z", "quotas": None, }, { diff --git a/backend/test/test_profiles.py b/backend/test/test_profiles.py index dfb3c235e..6035ab1cb 100644 --- a/backend/test/test_profiles.py +++ b/backend/test/test_profiles.py @@ -121,14 +121,20 @@ def profile_config_id(admin_auth_headers, default_org_id, profile_id): assert data["userid"] assert data["oid"] == default_org_id assert data.get("origins") or data.get("origins") == [] - assert data["created"] assert data["createdBy"] assert data["createdByName"] == "admin" - assert data["modified"] assert data["modifiedBy"] assert data["modifiedByName"] == "admin" assert not data["baseid"] + created = data["created"] + assert created + assert created.endswith("Z") + + modified = data["modified"] + assert modified + assert modified.endswith("Z") + resource = data["resource"] assert resource assert resource["filename"] diff --git a/backend/test/test_run_crawl.py b/backend/test/test_run_crawl.py index 9d2d4c940..d794ee9c9 100644 --- a/backend/test/test_run_crawl.py +++ b/backend/test/test_run_crawl.py @@ -620,13 +620,16 @@ def test_crawl_stats(crawler_auth_headers, default_org_id): assert row["state"] assert row["userid"] assert row["user"] - assert row["started"] assert row["finished"] or row["finished"] is None assert row["duration"] or row["duration"] == 0 assert row["pages"] or row["pages"] == 0 assert row["filesize"] or row["filesize"] == 0 assert row["avg_page_time"] or row["avg_page_time"] == 0 + started = row["started"] + assert started + assert started.endswith("Z") + def test_crawl_pages(crawler_auth_headers, default_org_id, crawler_crawl_id): # Test GET list endpoint @@ -777,9 +780,12 @@ def test_crawl_pages(crawler_auth_headers, default_org_id, crawler_crawl_id): assert page["notes"] == [] assert page["userid"] - assert page["modified"] assert page["approved"] + modified = page["modified"] + assert modified + assert modified.endswith("Z") + # Set approved to False and test filter again r = requests.patch( f"{API_PREFIX}/orgs/{default_org_id}/crawls/{crawler_crawl_id}/pages/{page_id}", diff --git a/frontend/src/components/orgs-list.ts b/frontend/src/components/orgs-list.ts index 1d6198a79..e10d37f2b 100644 --- a/frontend/src/components/orgs-list.ts +++ b/frontend/src/components/orgs-list.ts @@ -506,7 +506,7 @@ export class OrgsList extends BtrixElement { html` html` RelativeDuration.humanize( (crawl.finished - ? new Date(`${crawl.finished}Z`) + ? new Date(crawl.finished) : new Date() - ).valueOf() - new Date(`${crawl.started}Z`).valueOf(), + ).valueOf() - new Date(crawl.started).valueOf(), ), )} diff --git a/frontend/src/features/browser-profiles/select-browser-profile.ts b/frontend/src/features/browser-profiles/select-browser-profile.ts index 699e68ae0..1cf1f32eb 100644 --- a/frontend/src/features/browser-profiles/select-browser-profile.ts +++ b/frontend/src/features/browser-profiles/select-browser-profile.ts @@ -88,7 +88,7 @@ export class SelectBrowserProfile extends LiteElement {
${msg( str`in ${RelativeDuration.humanize( - new Date(`${workflow.lastCrawlTime}Z`).valueOf() - - new Date(`${workflow.lastCrawlStartTime}Z`).valueOf(), + new Date(workflow.lastCrawlTime).valueOf() - + new Date(workflow.lastCrawlStartTime).valueOf(), { compact: true }, )}`, )}`; @@ -295,7 +295,7 @@ export class WorkflowListItem extends LitElement { if (workflow.lastCrawlStartTime) { const diff = new Date().valueOf() - - new Date(`${workflow.lastCrawlStartTime}Z`).valueOf(); + new Date(workflow.lastCrawlStartTime).valueOf(); if (diff < 1000) { return ""; } @@ -371,7 +371,7 @@ export class WorkflowListItem extends LitElement { (workflow) => html` ${this.item!.finished ? html`${RelativeDuration.humanize( - new Date(`${this.item!.finished}Z`).valueOf() - - new Date(`${this.item!.started}Z`).valueOf(), + new Date(this.item!.finished).valueOf() - + new Date(this.item!.started).valueOf(), )}` : html` diff --git a/frontend/src/pages/org/archived-item-detail/ui/qa.ts b/frontend/src/pages/org/archived-item-detail/ui/qa.ts index ecf779259..97de13a84 100644 --- a/frontend/src/pages/org/archived-item-detail/ui/qa.ts +++ b/frontend/src/pages/org/archived-item-detail/ui/qa.ts @@ -344,7 +344,7 @@ export class ArchivedItemDetailQA extends BtrixElement { html` msg( str`${workflow.createdByName} on ${this.dateFormatter.format( - new Date(`${workflow.created}Z`), + new Date(workflow.created), )}`, ), )} @@ -951,7 +951,7 @@ export class WorkflowDetail extends LiteElement { this.lastCrawlStartTime ? RelativeDuration.humanize( new Date().valueOf() - - new Date(`${this.lastCrawlStartTime}Z`).valueOf(), + new Date(this.lastCrawlStartTime).valueOf(), ) : skeleton, )} @@ -1589,8 +1589,7 @@ export class WorkflowDetail extends LiteElement { }, ); this.lastCrawlId = data.started; - // remove 'Z' from timestamp to match API response - this.lastCrawlStartTime = new Date().toISOString().slice(0, -1); + this.lastCrawlStartTime = new Date().toISOString(); this.logs = undefined; void this.fetchWorkflow(); this.goToTab("watch");