diff --git a/bedrock-devnet/devnet/__init__.py b/bedrock-devnet/devnet/__init__.py index d92d94e340ff..b2ae71e5f14c 100644 --- a/bedrock-devnet/devnet/__init__.py +++ b/bedrock-devnet/devnet/__init__.py @@ -179,6 +179,16 @@ def devnet_l2_allocs(paths): # Bring up the devnet where the contracts are deployed to L1 def devnet_deploy(paths): + + log.info('Starting peripheral telemetry services.') + run_command(['docker', 'compose', 'up', '-d', 'prometheus'], cwd=paths.ops_bedrock_dir, env={ + 'PWD': paths.ops_bedrock_dir + }) + + run_command(['docker', 'compose', 'up', '-d', 'grafana'], cwd=paths.ops_bedrock_dir, env={ + 'PWD': paths.ops_bedrock_dir + }) + if os.path.exists(paths.genesis_l1_path): log.info('L1 genesis already generated.') else: diff --git a/ops-bedrock/docker-compose.yml b/ops-bedrock/docker-compose.yml index a204aa844690..ff0901db32f8 100644 --- a/ops-bedrock/docker-compose.yml +++ b/ops-bedrock/docker-compose.yml @@ -194,22 +194,41 @@ services: OP_CHALLENGER_NUM_CONFIRMATIONS: 1 da-server: - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/da-server:devnet - build: - context: ../ - dockerfile: ops/docker/op-stack-go/Dockerfile - target: da-server-target - command: > - da-server - --file.path=/data - --addr=0.0.0.0 - --port=3100 - --log.level=debug - --generic-commitment="${PLASMA_GENERIC_DA}" + platform: linux/amd64 + image: ghcr.io/layr-labs/eigenda-proxy@sha256:c7aae98c0ed65763e0bfe9b2f41147123d6d576f435dcc2d0d8dbcbb03cdc205 + environment: + - EIGENDA_PROXY_ADDR=0.0.0.0 + - EIGENDA_PROXY_PORT=3100 + - MEMSTORE_ENABLED=true + - MEMSTORE_EXPIRATION=45m + - EIGENDA_PROXY_SIGNER_PRIVATE_KEY_HEX=0000000000000000000100000000000000000000000000000000000000000000 + - EIGENDA_PROXY_EIGENDA_DISPERSER_RPC=disperser-holesky.eigenda.xyz:443 ports: - - "3100:3100" + - 3100:3100 + - 6969:7300 + + grafana: + image: grafana/grafana:11.1.0 + restart: unless-stopped + env_file: + - grafana.env volumes: - - "da_data:/data" + - ./monitoring/grafana/provisioning/:/etc/grafana/provisioning/:ro + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards + # - grafana_data:/var/lib/grafana + ports: + - 3000:3000 + + prometheus: + image: prom/prometheus:latest + restart: unless-stopped + env_file: + - .env + volumes: + - ./monitoring/prometheus:/etc/prometheus + # - prometheus_data:/prometheus + ports: + - 9090:9090 sentinel: image: quarry/sentinel # TODO(10141): We need a public image for this (sentinel is out of repo) diff --git a/ops-bedrock/grafana.env b/ops-bedrock/grafana.env new file mode 100644 index 000000000000..3a7446dfccfa --- /dev/null +++ b/ops-bedrock/grafana.env @@ -0,0 +1 @@ +GF_SECURITY_ADMIN_PASSWORD=admin diff --git a/ops-bedrock/monitoring/grafana/dashboards/op_stack_compose_dashboard.json b/ops-bedrock/monitoring/grafana/dashboards/op_stack_compose_dashboard.json new file mode 100644 index 000000000000..0ec4401eb5d9 --- /dev/null +++ b/ops-bedrock/monitoring/grafana/dashboards/op_stack_compose_dashboard.json @@ -0,0 +1,1023 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "panels": [], + "title": "Sequencer", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 54 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "op_node_default_refs_number{type=\"l2_unsafe\", layer=\"l2\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "L2 Unsafe Block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 54 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "op_node_default_refs_number{type=\"l2_safe\", layer=\"l2\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "L2 Safe Block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 54 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "op_node_default_refs_number{type=\"l2_finalized\", layer=\"l2\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "L2 Finalized Block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 54 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "op_node_default_refs_number{type=\"l1_head\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "L1 Head", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 54 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "op_node_default_refs_number{type=\"l1_safe\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "L1 Safe Block", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 54 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "op_node_default_refs_number{type=\"l1_finalized\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "L1 Finalized Block", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 5, + "panels": [], + "title": "Balances", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 4, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 10 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 54 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "op_batcher_default_balance", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Batcher Balance", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Balance" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 10 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hoverProximity": -46, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "op_batcher_default_balance", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Batcher Balance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Balance" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 10 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hoverProximity": -46, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(op_node_default_l1_reorg_depth_count[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "L1 Reorg Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 4, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 14 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 54 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "op_proposer_default_balance", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Proposer Balance", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "displayName", + "value": "Balance" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 18 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "op_proposer_default_balance", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Proposer Balance", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "OP Stack Compose", + "uid": "ddshn0bgt86ioc", + "version": 1, + "weekStart": "" +} diff --git a/ops-bedrock/monitoring/grafana/provisioning/dashboards/all.yml b/ops-bedrock/monitoring/grafana/provisioning/dashboards/all.yml new file mode 100644 index 000000000000..e281a1e592c8 --- /dev/null +++ b/ops-bedrock/monitoring/grafana/provisioning/dashboards/all.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: true + editable: true + options: + path: /var/lib/grafana/dashboards diff --git a/ops-bedrock/monitoring/grafana/provisioning/datasources/all.yml b/ops-bedrock/monitoring/grafana/provisioning/datasources/all.yml new file mode 100644 index 000000000000..af7179f3f863 --- /dev/null +++ b/ops-bedrock/monitoring/grafana/provisioning/datasources/all.yml @@ -0,0 +1,15 @@ +apiVersion: 1 + +deleteDatasources: +- name: 'Prometheus' + +datasources: +- access: 'proxy' + editable: true + is_default: true + name: 'Prometheus' + uid: 'ddshms3dlineoe' + org_id: 1 + type: 'prometheus' + url: 'http://prometheus:9090' + version: 1 \ No newline at end of file diff --git a/ops-bedrock/monitoring/prometheus/prometheus.yml b/ops-bedrock/monitoring/prometheus/prometheus.yml new file mode 100644 index 000000000000..e3efe9f5f4ed --- /dev/null +++ b/ops-bedrock/monitoring/prometheus/prometheus.yml @@ -0,0 +1,30 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'l1-geth' + static_configs: + - targets: ['l1:6060'] + - job_name: 'l2-geth' + static_configs: + - targets: ['l2:6060'] + - job_name: 'op-node' + static_configs: + - targets: ['op-node:7300'] + + - job_name: 'op-batcher' + static_configs: + - targets: ['op-batcher:7300'] + + - job_name: 'op-proposer' + static_configs: + - targets: ['op-proposer:7300'] + + - job_name: 'op-geth' + static_configs: + - targets: ['op-geth:7300'] + + - job_name: 'eigenda-proxy' + static_configs: + - targets: ['da-server:6969']