From 27ca5f7a89c9260b8b3e951eda2d551841191101 Mon Sep 17 00:00:00 2001 From: zack <6351754+zackkrida@users.noreply.github.com> Date: Wed, 22 May 2024 06:46:59 -0400 Subject: [PATCH 1/8] Install caniuse-lite as a frontend dev dependency (#4368) --- frontend/package.json | 1 + pnpm-lock.yaml | 28 +++++++++++++--------------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 650ce1bd13b..0f5f5572f38 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -122,6 +122,7 @@ "babel-core": "^7.0.0-bridge.0", "babel-jest": "^29.7.0", "babel-loader": "8.3.0", + "caniuse-lite": "^1.0.30001620", "chokidar": "^3.5.3", "comment-json": "^4.2.3", "css-loader": "^5.2.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6ac3e1ad4af..cb6a4a8b385 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -243,6 +243,9 @@ importers: babel-loader: specifier: 8.3.0 version: 8.3.0(@babel/core@7.24.4)(webpack@4.46.0) + caniuse-lite: + specifier: ^1.0.30001620 + version: 1.0.30001620 chokidar: specifier: ^3.5.3 version: 3.5.3 @@ -3918,11 +3921,8 @@ packages: caniuse-api@3.0.0: resolution: {integrity: sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw==} - caniuse-lite@1.0.30001554: - resolution: {integrity: sha512-A2E3U//MBwbJVzebddm1YfNp7Nud5Ip+IPn4BozBmn4KqVX7AvluoIDFWjsv5OkGnKUXQVmMSoMKLa3ScCblcQ==} - - caniuse-lite@1.0.30001606: - resolution: {integrity: sha512-LPbwnW4vfpJId225pwjZJOgX1m9sGfbw/RKJvw/t0QhYOOaTXHvkjVGFGPpvwEzufrjvTlsULnVTxdy4/6cqkg==} + caniuse-lite@1.0.30001620: + resolution: {integrity: sha512-WJvYsOjd1/BYUY6SNGUosK9DUidBPDTnOARHp3fSmFO1ekdxaY6nKRttEVrfMmYi80ctS0kz1wiWmm14fVc3ew==} capture-exit@2.0.0: resolution: {integrity: sha512-PiT/hQmTonHhl/HFGN+Lx3JJUznrVYJ3+AQsnthneZbvW7x+f08Tk7yLJTLEOUvBTbduLeeBkxEaYXUOUrRq6g==} @@ -13263,7 +13263,7 @@ snapshots: '@nuxt/utils': 2.17.3 babel-loader: 8.3.0(@babel/core@7.24.4)(webpack@4.47.0) cache-loader: 4.1.0(webpack@4.47.0) - caniuse-lite: 1.0.30001606 + caniuse-lite: 1.0.30001620 consola: 3.2.3 css-loader: 5.2.7(webpack@4.47.0) cssnano: 6.1.2(postcss@8.4.38) @@ -15948,7 +15948,7 @@ snapshots: autoprefixer@10.4.16(postcss@8.4.31): dependencies: browserslist: 4.22.1 - caniuse-lite: 1.0.30001554 + caniuse-lite: 1.0.30001620 fraction.js: 4.3.7 normalize-range: 0.1.2 picocolors: 1.0.0 @@ -15958,7 +15958,7 @@ snapshots: autoprefixer@10.4.16(postcss@8.4.38): dependencies: browserslist: 4.22.1 - caniuse-lite: 1.0.30001554 + caniuse-lite: 1.0.30001620 fraction.js: 4.3.7 normalize-range: 0.1.2 picocolors: 1.0.0 @@ -15968,7 +15968,7 @@ snapshots: autoprefixer@9.8.8: dependencies: browserslist: 4.23.0 - caniuse-lite: 1.0.30001606 + caniuse-lite: 1.0.30001620 normalize-range: 0.1.2 num2fraction: 1.2.2 picocolors: 0.2.1 @@ -16293,14 +16293,14 @@ snapshots: browserslist@4.22.1: dependencies: - caniuse-lite: 1.0.30001554 + caniuse-lite: 1.0.30001620 electron-to-chromium: 1.4.568 node-releases: 2.0.13 update-browserslist-db: 1.0.13(browserslist@4.22.1) browserslist@4.23.0: dependencies: - caniuse-lite: 1.0.30001606 + caniuse-lite: 1.0.30001620 electron-to-chromium: 1.4.729 node-releases: 2.0.14 update-browserslist-db: 1.0.13(browserslist@4.23.0) @@ -16461,13 +16461,11 @@ snapshots: caniuse-api@3.0.0: dependencies: browserslist: 4.23.0 - caniuse-lite: 1.0.30001606 + caniuse-lite: 1.0.30001620 lodash.memoize: 4.1.2 lodash.uniq: 4.5.0 - caniuse-lite@1.0.30001554: {} - - caniuse-lite@1.0.30001606: {} + caniuse-lite@1.0.30001620: {} capture-exit@2.0.0: dependencies: From 9b27da85ba4f26db0b5b445145c36c373124ddf6 Mon Sep 17 00:00:00 2001 From: zack <6351754+zackkrida@users.noreply.github.com> Date: Wed, 22 May 2024 16:01:55 -0400 Subject: [PATCH 2/8] Add frontend media documentation (#4313) Co-authored-by: Olga Bulat --- .../meta/media_properties/frontend.md | 283 +++++++++++++++--- frontend/src/types/media.ts | 127 +++++++- 2 files changed, 354 insertions(+), 56 deletions(-) diff --git a/documentation/meta/media_properties/frontend.md b/documentation/meta/media_properties/frontend.md index 2d2017ad4ca..7be76fa8d1d 100644 --- a/documentation/meta/media_properties/frontend.md +++ b/documentation/meta/media_properties/frontend.md @@ -22,38 +22,102 @@ clarity. ### Fields -| Name | Type | Optional? | -| --------------------------------------------- | ----------------------------- | --------- | -| `attribution` | `string` | | -| `category` | `string \| null` | | -| `creator` | `string` | ✓ | -| `creator_url` | `string` | ✓ | -| `description` | `string` | ✓ | -| `detail_url` | `string` | | -| `fields_matched` | `string[]` | ✓ | -| `filesize` | `string` | ✓ | -| `filetype` | `string` | ✓ | -| `foreign_landing_url` | `string` | | -| `frontendMediaType` | `SupportedMediaType` (custom) | | -| [`id`](#Media-id-notes) | `string` | | -| `isSensitive` | `boolean` | | -| `license` | `License` (custom) | | -| `license_url` | `string` | ✓ | -| `license_version` | `LicenseVersion` (custom) | | -| [`originalTitle`](#Media-originalTitle-notes) | `string` | | -| `provider` | `string` | | -| `providerName` | `string` | | -| `related_url` | `string` | | -| `sensitivity` | `Sensitivity[]` (custom) | | -| `source` | `string` | | -| `sourceName` | `string` | | -| `tags` | `Tag[]` (custom) | | -| `thumbnail` | `string` | ✓ | -| [`title`](#Media-title-notes) | `string` | | -| `url` | `string` | | +| Name | Type | Optional? | +| --------------------------------------------------------- | ----------------------------- | --------- | +| [`attribution`](#Media-attribution-notes) | `string` | | +| [`category`](#Media-category-notes) | `string \| null` | ✓ | +| [`creator`](#Media-creator-notes) | `string` | ✓ | +| [`creator_url`](#Media-creator_url-notes) | `string` | ✓ | +| [`description`](#Media-description-notes) | `string` | ✓ | +| [`detail_url`](#Media-detail_url-notes) | `string` | | +| [`fields_matched`](#Media-fields_matched-notes) | `string[]` | ✓ | +| [`filesize`](#Media-filesize-notes) | `string` | ✓ | +| [`filetype`](#Media-filetype-notes) | `string` | ✓ | +| [`foreign_landing_url`](#Media-foreign_landing_url-notes) | `string` | | +| `frontendMediaType` | `SupportedMediaType` (custom) | | +| [`id`](#Media-id-notes) | `string` | | +| [`isSensitive`](#Media-isSensitive-notes) | `boolean` | | +| [`license`](#Media-license-notes) | `License` (custom) | | +| [`license_url`](#Media-license_url-notes) | `string` | ✓ | +| [`license_version`](#Media-license_version-notes) | `LicenseVersion` (custom) | | +| [`originalTitle`](#Media-originalTitle-notes) | `string` | | +| [`provider`](#Media-provider-notes) | `string` | | +| [`providerName`](#Media-providerName-notes) | `string` | | +| [`related_url`](#Media-related_url-notes) | `string` | | +| [`sensitivity`](#Media-sensitivity-notes) | `Sensitivity[]` (custom) | | +| [`source`](#Media-source-notes) | `string` | | +| [`sourceName`](#Media-sourceName-notes) | `string` | | +| [`tags`](#Media-tags-notes) | `Tag[]` (custom) | | +| [`thumbnail`](#Media-thumbnail-notes) | `string` | ✓ | +| [`title`](#Media-title-notes) | `string` | | +| [`url`](#Media-url-notes) | `string` | | ### Notes +(Media-attribution-notes)= + +#### `attribution` + +The full text to properly attribute the work to the creator. + +(Media-category-notes)= + +#### `category` + +The descriptive category describing the work. + +(Media-creator-notes)= + +#### `creator` + +The text name, handle, or username of the author of the creative work. + +(Media-creator_url-notes)= + +#### `creator_url` + +A URL to the creator's profile on the provider or other personal webpage, +depending on the provider. + +(Media-description-notes)= + +#### `description` + +A long test describing the media, from the provider. + +(Media-detail_url-notes)= + +#### `detail_url` + +The API url of the detail view of a media. + +(Media-fields_matched-notes)= + +#### `fields_matched` + +An array of field names that matched the search query. + +(Media-filesize-notes)= + +#### `filesize` + +A number representing the size of the media in bytes. + +(Media-filetype-notes)= + +#### `filetype` + +A string representing the filetype of the media. Please note this is not an +exact representation of extension or MIME type. + +(Media-foreign_landing_url-notes)= + +#### `foreign_landing_url` + +A URL to the page where the media item is hosted on the foreign provider's site. +This is often used to give credit to the original source or for users to find +more information. + (Media-id-notes)= #### `id` @@ -64,11 +128,95 @@ the UUID4 identifier of the media item - [UUID4]() +(Media-isSensitive-notes)= + +#### `isSensitive` + +Indicates if the media is marked as sensitive. If true, the media has +sensitivity markers that might require user discretion or warnings. + +(Media-license-notes)= + +#### `license` + +The code of the open copy-left license assigned to the work. + +(Media-license_url-notes)= + +#### `license_url` + +A link to the landing page of the License, typically the deed or another +informational page. + +(Media-license_version-notes)= + +#### `license_version` + +The version number of the Creative Commons license as a string, or an empty +string. + (Media-originalTitle-notes)= #### `originalTitle` -the raw name of the creative work, as returned by the API +The raw name of the creative work, as returned by the API. + +(Media-provider-notes)= + +#### `provider` + +A snake_cased slug representing the media's provider. Corresponds to the +`source_name` field of provider results from the API's `/stats/` endpoints. + +(Media-providerName-notes)= + +#### `providerName` + +A presentational string (with capitalization, spaces, punctuation, and so on), +representing the media's provider. Corresponds to the `display_name` field of +provider results from the API's `/stats/` endpoints. + +(Media-related_url-notes)= + +#### `related_url` + +The API URL which provides a list of related media results. + +(Media-sensitivity-notes)= + +#### `sensitivity` + +An array of sensitivity markers. These markers indicate various sensitivity or +content warning categories applicable to the media. + +(Media-source-notes)= + +#### `source` + +A snake_cased slug representing the media's source. Corresponds to the +`source_name` field of provider results from the API's `/stats/` endpoints. + +(Media-sourceName-notes)= + +#### `sourceName` + +A presentational string (with capitalization, spaces, punctuation, and so on), +representing the media's source. Corresponds to the `display_name` field of +provider results from the API's `/stats/` endpoints. + +(Media-tags-notes)= + +#### `tags` + +An array of tags, used to query the media or present on the frontend to find +related media. + +(Media-thumbnail-notes)= + +#### `thumbnail` + +A URL to a thumbnail image of the media. Typically album art for an audio track, +or a minified thumbnail for image works. (Media-title-notes)= @@ -81,6 +229,12 @@ the original title: - escape HTML - handle empty titles +(Media-url-notes)= + +#### `url` + +A URL pointing to the actual media file on the provider. + ## Interface `ImageDetail` ### Fields @@ -97,13 +251,13 @@ the original title: #### `height` -the vertical length of the image in pixels +The vertical length of the image, in pixels. (ImageDetail-width-notes)= #### `width` -the horizontal length of the image in pixels +The horizontal length of the image, in pixels. ## Interface `AudioDetail` @@ -111,26 +265,39 @@ the horizontal length of the image in pixels | Name | Type | Optional? | | ----------------------------------------------- | ------------------------------------------ | --------- | -| `alt_files` | `{ provider: string; filetype: string }[]` | ✓ | -| `audio_set` | `AudioSet` (custom) | ✓ | +| [`alt_files`](#AudioDetail-alt_files-notes) | `{ provider: string; filetype: string }[]` | ✓ | +| [`audio_set`](#AudioDetail-audio_set-notes) | `AudioSet` (custom) | ✓ | | [`bit_rate`](#AudioDetail-bit_rate-notes) | `number` | ✓ | | [`duration`](#AudioDetail-duration-notes) | `number` | ✓ | | `frontendMediaType` | `"audio"` (literal) | | -| `genres` | `string[]` | ✓ | -| `hasLoaded` | `boolean` | ✓ | -| `length` | `string` | ✓ | -| `peaks` | `number[]` | ✓ | +| [`genres`](#AudioDetail-genres-notes) | `string[]` | ✓ | +| [`hasLoaded`](#AudioDetail-hasLoaded-notes) | `boolean` | ✓ | +| [`peaks`](#AudioDetail-peaks-notes) | `number[]` | ✓ | | [`sample_rate`](#AudioDetail-sample_rate-notes) | `number` | ✓ | -| `waveform` | `string` | ✓ | +| [`waveform`](#AudioDetail-waveform-notes) | `string` | ✓ | ### Notes +(AudioDetail-alt_files-notes)= + +#### `alt_files` + +An array of alternative files of different filetypes. This can include different +formats of the audio track for compatibility and quality options. + +(AudioDetail-audio_set-notes)= + +#### `audio_set` + +An object representing a group of audio tracks this track belongs to, like an +album or podcast. + (AudioDetail-bit_rate-notes)= #### `bit_rate` -amount of digital audio data transmitted or processed in unit time; This field -holds numbers measured in bits per second. +Amount of digital audio data transmitted or processed in unit time. This field +holds numbers measured in bits per second (bps). **See also:** @@ -140,15 +307,43 @@ holds numbers measured in bits per second. #### `duration` -the time period of the track in milliseconds +The duration of the track in milliseconds. + +(AudioDetail-genres-notes)= + +#### `genres` + +A raw list of strings representing musical genres. + +(AudioDetail-hasLoaded-notes)= + +#### `hasLoaded` + +Set and managed by the frontend client-side to indicate if the audio has been +fully loaded. Useful for managing UI elements based on the loading state of the +audio. + +(AudioDetail-peaks-notes)= + +#### `peaks` + +An array of peak amplitude values used to generate a visual representation of +the audio waveform. (AudioDetail-sample_rate-notes)= #### `sample_rate` -number of samples for digital representation taken in unit time; This field -holds numbers measured in hertz. +Number of samples for digital representation taken in unit time. This field +holds numbers measured in hertz (Hz). **See also:** - [Wikipedia]() + +(AudioDetail-waveform-notes)= + +#### `waveform` + +The URL of the API `/wavepoint` endpoint for the audio track. This endpoint +provides the full peaks array for the track. diff --git a/frontend/src/types/media.ts b/frontend/src/types/media.ts index e23e97b9823..a783cce68f4 100644 --- a/frontend/src/types/media.ts +++ b/frontend/src/types/media.ts @@ -26,6 +26,7 @@ export interface Media { * @see {@link https://en.wikipedia.org/wiki/Universally_unique_identifier#Version_4_(random)|UUID4} */ id: string + /** * the name of the creative work; This involves the following kinds of * changes to the original title: @@ -35,50 +36,133 @@ export interface Media { * - handle empty titles */ title: string - /** the raw name of the creative work, as returned by the API */ + + /** The raw name of the creative work, as returned by the API. */ originalTitle: string + /** The text name, handle, or username of the author of the creative work. */ creator?: string + + /** A URL to the creator's profile on the provider or other personal webpage, depending on the provider. */ creator_url?: string + /** A URL pointing to the actual media file on the provider. */ url: string + + /** + * A URL to the page where the media item is hosted on the foreign provider's site. + * This is often used to give credit to the original source or for users to find more information. + */ foreign_landing_url: string + /** The code of the open copy-left license assigned to the work. */ license: License + + /** The version number of the Creative Commons license as a string, or an empty string. */ license_version: LicenseVersion + + /** A link to the landing page of the License, typically the deed or another informational page. */ license_url?: string + + /** The full text to properly attribute the work to the creator. */ attribution: string frontendMediaType: SupportedMediaType + /** A long test describing the media, from the provider. */ description?: string - category: string | null + /** The descriptive category describing the work. */ + category?: string | null + + /** + * A snake_cased slug representing the media's provider. + * + * Corresponds to the `source_name` field of provider results + * from the API's `/stats/` endpoints. + */ provider: string + + /** + * A snake_cased slug representing the media's source. + * + * Corresponds to the `source_name` field of provider results + * from the API's `/stats/` endpoints. + */ source: string + + /** + * A presentational string (with capitalization, spaces, punctuation, and so on), + * representing the media's provider. + * + * Corresponds to the `display_name` field of provider results + * from the API's `/stats/` endpoints. + */ providerName: string + + /** + * A presentational string (with capitalization, spaces, punctuation, and so on), + * representing the media's source. + * + * Corresponds to the `display_name` field of provider results + * from the API's `/stats/` endpoints. + */ sourceName: string + + /** + * A URL to a thumbnail image of the media. + * + * Typically album art for an audio track, or + * a minified thumbnail for image works. + */ thumbnail?: string + /** + * A number representing the size of the media in bytes. + */ filesize?: string + + /** + * A string representing the filetype of the media. + * Please note this is not an exact representation of extension or MIME type. + */ filetype?: string + /** + * The API url of the detail view of a media. + */ detail_url: string + + /** The API URL which provides a list of related media results. */ related_url: string + /** + * An array of tags, used to query the media or present on the frontend + * to find related media. + */ tags: Tag[] + + /** An array of field names that matched the search query. */ fields_matched?: string[] + /** + * An array of sensitivity markers. + * These markers indicate various sensitivity or content warning categories applicable to the media. + */ sensitivity: Sensitivity[] + + /** + * Indicates if the media is marked as sensitive. + * If true, the media has sensitivity markers that might require user discretion or warnings. + */ isSensitive: boolean } export interface ImageDetail extends Media { frontendMediaType: "image" - - /** the vertical length of the image in pixels */ + /** The vertical length of the image, in pixels. */ height?: number - /** the horizontal length of the image in pixels */ + /** The horizontal length of the image, in pixels. */ width?: number } @@ -95,30 +179,49 @@ export interface AudioSet { export interface AudioDetail extends Media { frontendMediaType: "audio" + /** An object representing a group of audio tracks this track belongs to, like an album or podcast. */ audio_set?: AudioSet + + /** A raw list of strings representing musical genres. */ genres?: string[] - length?: string - /** the time period of the track in milliseconds */ + + /** + * The duration of the track in milliseconds. + */ duration?: number + /** - * amount of digital audio data transmitted or processed in unit time; This - * field holds numbers measured in bits per second. + * Amount of digital audio data transmitted or processed in unit time. + * This field holds numbers measured in bits per second (bps). * * @see {@link https://en.wikipedia.org/wiki/Bit_rate#Audio|Wikipedia} */ bit_rate?: number + /** - * number of samples for digital representation taken in unit time; This field - * holds numbers measured in hertz. + * Number of samples for digital representation taken in unit time. + * This field holds numbers measured in hertz (Hz). * * @see {@link https://en.wikipedia.org/wiki/Sampling_(signal_processing)#Audio_sampling|Wikipedia} */ sample_rate?: number + + /** + * An array of alternative files of different filetypes. + * This can include different formats of the audio track for compatibility and quality options. + */ alt_files?: { provider: string; filetype: string }[] + + /** An array of peak amplitude values used to generate a visual representation of the audio waveform. */ peaks?: number[] + + /** The URL of the API `/wavepoint` endpoint for the audio track. This endpoint provides the full peaks array for the track. */ waveform?: string - // Set and managed by the frontend client-side + /** + * Set and managed by the frontend client-side to indicate if the audio has been fully loaded. + * Useful for managing UI elements based on the loading state of the audio. + */ hasLoaded?: boolean } From 0e2ed03c34c134a36988e8dfd1925c220d709c03 Mon Sep 17 00:00:00 2001 From: Darien Springer <32495526+thegreendrinker@users.noreply.github.com> Date: Thu, 23 May 2024 04:59:42 -0700 Subject: [PATCH 3/8] Use `.venv` for catalog virtualenv instead of `venv` (#4369) --- catalog/justfile | 2 +- documentation/catalog/guides/quickstart.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/catalog/justfile b/catalog/justfile index 30777f754d5..cb6bffcabb2 100644 --- a/catalog/justfile +++ b/catalog/justfile @@ -34,7 +34,7 @@ export CATALOG_AIRFLOW_VERSION := `grep '^apache-airflow' requirements-prod.txt # Create a virtual environment using the project Python version venv: # Invokes `python`, like `python3.10` - bash -c "python$CATALOG_PY_VERSION -m venv venv/" + bash -c "python$CATALOG_PY_VERSION -m venv .venv/" # Check that the active Python version matches the required Python version check-py-version: diff --git a/documentation/catalog/guides/quickstart.md b/documentation/catalog/guides/quickstart.md index 02488e3b9b0..08b7763c436 100644 --- a/documentation/catalog/guides/quickstart.md +++ b/documentation/catalog/guides/quickstart.md @@ -52,8 +52,8 @@ To set up the local python environment along with the pre-commit hook, run: ```shell -python3 -m venv venv -source venv/bin/activate +just venv +source .venv/bin/activate just catalog/install ``` From 925106b37f4d34d0a6ccce481883c9d3644b2811 Mon Sep 17 00:00:00 2001 From: Olga Bulat Date: Thu, 23 May 2024 18:01:44 +0300 Subject: [PATCH 4/8] Only set the user-agent header on the server (#4375) * Only set the user-agent header on server Signed-off-by: Olga Bulat * Update frontend/src/data/api-service.ts Co-authored-by: zack <6351754+zackkrida@users.noreply.github.com> * Lint and remove unused type Signed-off-by: Olga Bulat --------- Signed-off-by: Olga Bulat Co-authored-by: zack <6351754+zackkrida@users.noreply.github.com> --- frontend/src/data/api-service.ts | 44 +++++++++++++++----------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/frontend/src/data/api-service.ts b/frontend/src/data/api-service.ts index f67bc102484..eff1d79b31d 100644 --- a/frontend/src/data/api-service.ts +++ b/frontend/src/data/api-service.ts @@ -4,17 +4,10 @@ import { warn } from "~/utils/console" import { AUDIO, IMAGE } from "~/constants/media" import { userAgent } from "~/constants/user-agent" +import { isServer } from "~/utils/node-env" const DEFAULT_REQUEST_TIMEOUT = 30000 -/** - * Openverse Axios request config with adjusted types for our use-case. - */ -export type OpenverseAxiosRequestConfig = Required< - Pick -> & - AxiosRequestConfig - /** * Returns a slug with trailing slash for a given resource name. * For media types, converts the name into resource slug when necessary (i.e. pluralizes 'image'), @@ -34,7 +27,7 @@ export const getResourceSlug = (resource: string): string => { const validateRequest = ( errorCondition: boolean, message: string, - config: OpenverseAxiosRequestConfig + config: AxiosRequestConfig ): void => { if (errorCondition) { warn( @@ -73,22 +66,22 @@ export interface ApiService { post( resource: string, data: Parameters[1], - headers?: OpenverseAxiosRequestConfig["headers"] + headers?: AxiosRequestConfig["headers"] ): Promise> update( resource: string, slug: string, data: Parameters[1], - headers: OpenverseAxiosRequestConfig["headers"] + headers: AxiosRequestConfig["headers"] ): Promise> put( resource: string, - params: OpenverseAxiosRequestConfig + params: AxiosRequestConfig ): Promise> delete( resource: string, slug: string, - headers: OpenverseAxiosRequestConfig["headers"] + headers: AxiosRequestConfig["headers"] ): Promise> } @@ -97,14 +90,19 @@ export const createApiService = ({ accessToken = undefined, isVersioned = true, }: ApiServiceConfig = {}): ApiService => { - const axiosParams: OpenverseAxiosRequestConfig = { - baseURL: isVersioned ? `${baseUrl}v1/` : baseUrl, - timeout: DEFAULT_REQUEST_TIMEOUT, - headers: { "User-Agent": userAgent }, + const headers: AxiosRequestConfig["headers"] = {} + + if (isServer) { + headers["User-Agent"] = userAgent } if (accessToken) { - axiosParams.headers["Authorization"] = `Bearer ${accessToken}` + headers["Authorization"] = `Bearer ${accessToken}` + } + const axiosParams: AxiosRequestConfig = { + baseURL: isVersioned ? `${baseUrl}v1/` : baseUrl, + timeout: DEFAULT_REQUEST_TIMEOUT, + headers, } const client = axios.create(axiosParams) @@ -112,12 +110,12 @@ export const createApiService = ({ validateRequest( !config.url?.endsWith("/"), "API request urls should have a trailing slash", - config as OpenverseAxiosRequestConfig + config as AxiosRequestConfig ) validateRequest( config.url?.includes("//") ?? false, "API request urls should not have two slashes", - config as OpenverseAxiosRequestConfig + config as AxiosRequestConfig ) return config }) @@ -185,7 +183,7 @@ export const createApiService = ({ resource: string, slug: string, data: Parameters<(typeof client)["put"]>[1], - headers: OpenverseAxiosRequestConfig["headers"] + headers: AxiosRequestConfig["headers"] ): Promise> { return client.put(`${getResourceSlug(resource)}${slug}`, data, { headers, @@ -199,7 +197,7 @@ export const createApiService = ({ */ put( resource: string, - params: OpenverseAxiosRequestConfig + params: AxiosRequestConfig ): Promise> { return client.put(getResourceSlug(resource), params) }, @@ -213,7 +211,7 @@ export const createApiService = ({ delete( resource: string, slug: string, - headers: OpenverseAxiosRequestConfig["headers"] + headers: AxiosRequestConfig["headers"] ): Promise> { return client.delete(`${getResourceSlug(resource)}${slug}`, { headers }) }, From 4eeb3781a7335f837e8cf6951f85bec6488c5d17 Mon Sep 17 00:00:00 2001 From: "Openverse (Bot)" <101814513+openverse-bot@users.noreply.github.com> Date: Fri, 24 May 2024 14:42:18 +1000 Subject: [PATCH 5/8] Publish changelog for api-2024.05.23.15.02.00 (#4377) Co-authored-by: AetherUnbound <10214785+AetherUnbound@users.noreply.github.com> --- documentation/changelogs/api/2024.05.23.15.02.00.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 documentation/changelogs/api/2024.05.23.15.02.00.md diff --git a/documentation/changelogs/api/2024.05.23.15.02.00.md b/documentation/changelogs/api/2024.05.23.15.02.00.md new file mode 100644 index 00000000000..de44cb85968 --- /dev/null +++ b/documentation/changelogs/api/2024.05.23.15.02.00.md @@ -0,0 +1,12 @@ +# 2024.05.23.15.02.00 + +## Internal Improvements + +- Add 'revoked' field to ThrottledApplication to enable easily revoking access + to client applications violating openverse TOS + ([#4334](https://github.com/WordPress/openverse/pull/4334)) by @madewithkode + +## Bug Fixes + +- Change search query approach to include only available providers + ([#4238](https://github.com/WordPress/openverse/pull/4238)) by @krysal From 87880a0bd9e901d982a1b6dd03cb08fcebb36535 Mon Sep 17 00:00:00 2001 From: Olga Bulat Date: Fri, 24 May 2024 14:39:43 +0300 Subject: [PATCH 6/8] Implementation Plan: Machine-generated tags on the frontend (#4302) * Add the IP for machine generated tags on the frontend Signed-off-by: Olga Bulat * Update documentation/projects/proposals/rekognition_data/20240510-implementation_plan_machine_generated_tags_frontend.md Co-authored-by: Madison Swain-Bowden * Update documentation/projects/proposals/rekognition_data/20240510-implementation_plan_machine_generated_tags_frontend.md Co-authored-by: Madison Swain-Bowden * Add missing data Signed-off-by: Olga Bulat * Update the IP to NOT display tag accuracy/provider Signed-off-by: Olga Bulat * Update the wording Signed-off-by: Olga Bulat * Add approvals Signed-off-by: Olga Bulat --------- Signed-off-by: Olga Bulat Co-authored-by: Madison Swain-Bowden --- ...on_plan_machine_generated_tags_frontend.md | 196 ++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 documentation/projects/proposals/rekognition_data/20240510-implementation_plan_machine_generated_tags_frontend.md diff --git a/documentation/projects/proposals/rekognition_data/20240510-implementation_plan_machine_generated_tags_frontend.md b/documentation/projects/proposals/rekognition_data/20240510-implementation_plan_machine_generated_tags_frontend.md new file mode 100644 index 00000000000..7c782f82cbc --- /dev/null +++ b/documentation/projects/proposals/rekognition_data/20240510-implementation_plan_machine_generated_tags_frontend.md @@ -0,0 +1,196 @@ +# 2024-05-08 Implementation Plan: Machine-generated tags in the frontend + +**Author**: @obulat + + + + +## Reviewers + + + +- [x] @zackkrida +- [x] @AetherUnbound + +## Project links + + + +- [Project Thread](https://github.com/WordPress/openverse/issues/431) +- [Project Proposal](/projects/proposals/rekognition_data/20240320-project_proposal_rekognition_data.md) + +## Overview + + + +This implementation plan will describe how we plan to display the +machine-generated tags separately from the source tags on the single result +page. All tags will be clickable and lead to a collection view by the selected +tag. The tag collection view will not be affected as we will not show different +results for a machine-generated tag than the source tag. + +## Expected Outcomes + + + +When this work is completed, the single result page will display +machine-generated tags separately from the source tags. For machine-generated +tags, we will also display the tag provider and its accuracy. + +## Step-by-step plan + + + +1. [Create a wrapper component that displays the tags in two sections: source-provided and machine generated](#create-a-wrapper-vmediatags-component) +2. [Add a page that describes the machine-generated and source tags.](#add-about-tags-description-page) + +## Step details + + + +### Create a wrapper `VMediaTags` component + +This component will handle filtering the tags into two sets: machine-generated +and source-provided. It will also display the headings and the link to the +"About tags" page. A new child component, `VCollapsibleTagSection`, will handle +displaying of the tags in several rows, collapsing them if there are more than +three rows of tags. + +Currently, we deduplicate all tags to only display unique tags. When we display +the tags by generation type, there will probably be duplicates between the +source and the generated tags. However, each section individually will still be +normalized to prevent duplicates. + +The generated tags section will have a link to the "About tags" page. + +There is no need to create a separate component for machine-generated tags, the +slot in the `VTag` component will enable us to display the additional accuracy +information. + +### Add "About tags" description page + +We should add a page to explain the users the difference between where the +Openverse tags come from, the difference between the source and generated tags, +and a short description of the machine-generated tag providers. This page should +be linked from the Generated tags section. This page will not be added to the +header navigation. + +## Dependencies + +This project needs the API to add the `unstable__provider` property to tags in +media responses, which was implemented in +[Expose provider in the API tags response #4280](https://github.com/WordPress/openverse/pull/4280) + +### Feature flags + + + +The project will contain at most 2 PRs that can be reverted, so the feature +flags are not necessary. + +### Infrastructure + + + +No infrastructure changes will be necessary, we will only update the single +result pages of the Nuxt frontend. + +## Alternatives + + + +This project states that it is important to clearly distinguish the +source-generated tags from the machine-generated tags. + +In addition to tag name, the machine-generated tags provide additional data +points: their provider (currently, it will be "clarifai" and "rekognition") and +their accuracy. This project aims to describe this data in a technical way on +the "About tags" page, without displaying each individual tag's additional data, +to prevent from overloading the users with too much information. Once we have +more user data on how users relate to machine-generated tags, we will reconsider +displaying this data on the single result page. + +Two other options were considered for displaying the machine-generated tags, but +were discarded as they would be less user-friendly. + +1. Display the tag's provider and accuracy in a tooltip. This option would + provide the user all the information about the tag without cluttering the UI. + However, this option is not good for accessibility. Different browsers and + screen readers treat the tooltips differently. +2. Display one section per each machine-generated tag provider, and label + sections with relevant heading, e.g. "Machine-Generated Tags: Rekognition". + Each provider section would sort the tags in the order of their accuracy, and + each tag would display the tag name and the tag accuracy. This way, the user + would see the data without additional actions such as hovering. We have + enough room on the page to display this data in full. As a bonus, if a tag is + generated by multiple providers, the users would be able to see that + different providers generate the same tag, and the difference in accuracy + between the providers. This option wasn't selected because we don't yet have + enough user data to know if the users would find this information useful + instead of overwhelming. + +## Design + + + +The first iteration of the designs that use the first alternative option is +ready: +[Displaying machine-generated content](https://github.com/WordPress/openverse/issues/4192) +If we decide to go with the third alternative option, we will need to add the +provider headings and the accuracy to the tags. + +## Blockers + + + +Since we already have machine-generated tags available in the API, there are no +blockers for this work. + +## Accessibility + + + +Adding the "Source tags" and "Generated tags" headings is good for the +accessibility of the tag list. Both the sighted and the screen reader users will +understand what the list is for. It is always desirable to have visible label +next to a list of elements. + +The display of generated tags' provider and accuracy is, on the other hand, a +challenging problem from accessibility standpoint. Tooltips proposed in the +second alternative option are often described as inaccessible for mobile users +who do not have a hover action. MDN's section on tooltip best practices suggests +providing "clear labels and sufficient body text" instead of tooltips that hide +important information. + +If we decide to display the accuracy on the tag together with its name (option 2 +above), it is important to separate the tag name and its accuracy (with an +invisible, `sr-only`, dot, for instance) to make the screen readers add a pause +between them. + +## Rollback + + + +We can roll back this solution by reverting the PRs implementing this. + +## Localization + + + +The tag headings will be translated as usual. For the "About page" it is +important to split the text into shorter chunks to make them easier to handle +for translators. The eslint rule will make sure that this is done properly. From 6167905fe7bf5d96a802f351ffc2e8f40d771fff Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Fri, 24 May 2024 18:50:21 +0400 Subject: [PATCH 7/8] Remove overridden function that doesn't do anything over super (#4380) --- api/api/admin/media_report.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/api/api/admin/media_report.py b/api/api/admin/media_report.py index 91427a93913..ba5364d39e6 100644 --- a/api/api/admin/media_report.py +++ b/api/api/admin/media_report.py @@ -261,14 +261,6 @@ def change_view(self, request, object_id, form_url="", extra_context=None): extra_context=extra_context, ) - def render_change_form( - self, request, context, add=False, change=False, form_url="", obj=None - ): - context.update({"add": add, "change": change}) - return super().render_change_form( - request, context, add=add, change=change, form_url=form_url, obj=obj - ) - class ImageReportAdmin(MediaReportAdmin): media_type = "image" From 6e8d0bf6f88fe28ca058c90e5567453145aada67 Mon Sep 17 00:00:00 2001 From: zack <6351754+zackkrida@users.noreply.github.com> Date: Fri, 24 May 2024 11:53:28 -0400 Subject: [PATCH 8/8] Document retired node replacement in ES (#4326) Co-authored-by: Madison Swain-Bowden --- .../meta/maintenance/elasticsearch_cluster.md | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/documentation/meta/maintenance/elasticsearch_cluster.md b/documentation/meta/maintenance/elasticsearch_cluster.md index f299c262ab5..bc3d7e5d748 100644 --- a/documentation/meta/maintenance/elasticsearch_cluster.md +++ b/documentation/meta/maintenance/elasticsearch_cluster.md @@ -336,6 +336,69 @@ etc. The main focus should be understanding the new configuration and determining whether it's possible to test it locally. Otherwise, follow the steps for deploying a new cluster and switching the live services over to it. +## Node Replacements + +Occasionally, the underlying EC2 instance for one of our nodes will undergo a +scheduled decomission ("retirement") and need to be replaced beforehand. + +This procedure describes the process of replacing a single node, but could be +extrapolated to multiple nodes with relative ease. + +If a data refresh is scheduled during the timeframe of this procedure, it is +best to pause it for performance reasons. + +### Steps + +1. Use the Instance ID provided in the retirement notification to find the + "Private IPv4 address" of the instance. This value is the "Name" of the + ElasticSearch node. +1. Identify where the node is stored in our Terraform state. This will be useful + for manual state changes later in the process. Check each node until you find + the matching private IP. In a terminal, navigate to the appropriate + environment directory and run: + + ```bash + terraform state show "module.-elasticsearch-8-8-2.aws_instance.datanodes[0] + terraform state show "module.-elasticsearch-8-8-2.aws_instance.datanodes[1] + # ...continue as needed until the correct node is returned + ``` + + Replace the `` with the correct name for the current environment. Record + the final `module.-elasticsearch-8-8-2.aws_instance.datanodes[x]` index + for later use. + +1. In Terraform, increase the `data_node_count` for the relevant Elasticsearch + module by one. Here is an + [example commit](https://github.com/WordPress/openverse-infrastructure/pull/894/commits/4a827b786b1460aa89931d474db119d835784727) + with this change in production. Apply this change and wait for a new instance + to be provisioned and connected to the cluster. Record the "Public IPv4 DNS" + record of the instance for use in the next step. +1. Configure the created instance by running our ansible playbook to sync + Elasticsearch nodes. Supply the correct environment name to the command and + pass the DNS record from the previous step to the limit flag like so: + `just ansible/playbook elasticsearch/sync_config.yml -e apply=true -l ` +1. Use `just jh es ` to connect to the cluster, and send + `{ "transient":{ "cluster.routing.allocation.exclude.name": "" } }` + to the `/_cluster/settings` endpoint (in a GUI like Elasticvue or via `curl`) + to deallocate shards from the bad node. Be sure to replace `` + with the private IPv4 address identified in step 1. +1. The cluster will now relocate shards to the new node and from the retired + node. Wait for the cluster health to return to green. +1. Manually terminate the retired instance in the AWS console. +1. In Terraform: + - Decrease the `data_node_count` for the Elasticsearch module down by one. Do + not apply this yet. + - Remove the retired node from the Terraform state. This is a prerequisite to + moving the new node into the retired node’s position in the state. Use the + following command, replacing "x" with the index found in step 2. + `j tf prod state rm module.elasticsearch.aws_instance.elasticsearch-ec2-datanodes[x]` + - Move the newest node added in step 3 to the position of the retired node we + just removed. If the newest node was in position 3 and the old node was in + position 0, for example: + `j tf state mv module.elasticsearch.aws_instance.elasticsearch-ec2-datanodes[3] module.elasticsearch.aws_instance.elasticsearch-ec2-datanodes[0]` +1. Apply the Terraform changes, which will cleanup the dangling DNS records of + the retired node. + ## Potential improvements - Pass node configuration at the module level rather than generating it inside