Merge branch 'develop' into feat/export-dataset-to-hub

.github/workflows/argilla-frontend.yml

@@ -71,8 +71,6 @@ jobs:
       image-name: argilla/argilla-frontend-for-dev
       dockerfile: argilla-frontend/dev.frontend.Dockerfile
       platforms: linux/amd64
-      build-args: |
-        ARGILLA_SERVER_TAG=main
     secrets: inherit
 
   deploy:

argilla-frontend/components/features/annotation/container/fields/RecordFields.vue

@@ -17,7 +17,7 @@
     >
       <SpanAnnotationTextField
         v-if="hasSpanQuestion(name)"
-        :id="`${id}-${record.id}-span-field`"
+        :id="`${id}-${record.id}`"
         :name="name"
         :title="title"
         :fieldText="content"
@@ -26,7 +26,7 @@
       />
       <TextField
         v-else-if="isTextType"
-        :name="name"
+        :id="`${id}-${record.id}`"
         :title="title"
         :fieldText="content"
         :useMarkdown="settings.use_markdown"
@@ -35,7 +35,7 @@
       />
       <ChatField
         v-else-if="isChatType"
-        :name="name"
+        :id="`${id}-${record.id}`"
         :title="title"
         :useMarkdown="settings.use_markdown"
         :content="content"

argilla-frontend/components/features/annotation/container/fields/chat-field/ChatField.vue

@@ -2,7 +2,7 @@
   <div class="chat" :key="title">
     <span class="chat__title" v-text="title" />
     <div
-      :id="`fields-content-${name}`"
+      :id="`fields-content-${id}`"
       class="chat__wrapper"
       :class="checkIfAreLessThanTwoRoles ? '--simple' : '--multiple'"
     >
@@ -32,8 +32,8 @@
           >
             <MarkdownRenderer v-if="useMarkdown" :markdown="text" />
             <span v-else v-html="text" /><template>
-              <style :key="name" scoped>
-                ::highlight(search-text-highlight-{{name}}) {
+              <style :key="id" scoped>
+                ::highlight(search-text-highlight-{{id}}) {
                   color: #ff675f;
                 }
               </style>
@@ -47,9 +47,10 @@
 
 <script>
 import { useChatFieldViewModel } from "./useChatFieldViewModel";
+
 export default {
   props: {
-    name: {
+    id: {
       type: String,
       required: true,
     },

argilla-frontend/components/features/annotation/container/fields/chat-field/useChatFieldViewModel.ts

@@ -2,10 +2,10 @@ import { onMounted, watch } from "vue-demi";
 import { useSearchTextHighlight } from "../useSearchTextHighlight";
 
 export const useChatFieldViewModel = (props: {
-  name: string;
+  id: string;
   searchText: string;
 }) => {
-  const { highlightText } = useSearchTextHighlight(props.name);
+  const { highlightText } = useSearchTextHighlight(props.id);
 
   watch(
     () => props.searchText,

argilla-frontend/components/features/annotation/container/fields/span-annotation/SpanAnnotationTextField.vue

@@ -1,7 +1,7 @@
 <template>
   <div
     class="text_field_component"
-    :key="name"
+    :key="id"
     @mouseenter.stop="mouseEnter = true"
     @mouseleave.stop="mouseEnter = false"
     aria-label="Data Record Field"
@@ -35,7 +35,7 @@
       </BaseActionTooltip>
     </div>
     <div
-      :id="`fields-content-${name}`"
+      :id="`fields-content-${id}`"
       class="text_field_component__area --body1"
       :aria-label="'Data entry for Field: ' + title"
     >
@@ -97,7 +97,7 @@
             [data-theme="dark"] .span-annotation__field--overlapped::highlight(hl-{{id}}-hover) {
               background: {{color.palette.veryDark}};
             }
-            ::highlight(search-text-highlight-{{name}}) {
+            ::highlight(search-text-highlight-{{id}}) {
               color: #ff675f;
             }
           </style>
@@ -110,16 +110,11 @@
 <script>
 import { useSpanAnnotationTextFieldViewModel } from "./useSpanAnnotationTextFieldViewModel";
 export default {
-  name: "SpanAnnotationTextField",
   props: {
     id: {
       type: String,
       required: true,
     },
-    name: {
-      type: String,
-      required: true,
-    },
     title: {
       type: String,
       required: true,

argilla-frontend/components/features/annotation/container/fields/span-annotation/useSpanAnnotationTextFieldViewModel.ts

@@ -9,13 +9,12 @@ import { SpanQuestionAnswer } from "~/v1/domain/entities/question/QuestionAnswer
 import { SpanAnswer } from "~/v1/domain/entities/IAnswer";
 
 export const useSpanAnnotationTextFieldViewModel = (props: {
-  name: string;
   spanQuestion: Question;
   id: string;
   searchText: string;
 }) => {
-  const { name, spanQuestion, id } = props;
-  const searchTextHighlight = useSearchTextHighlight(name);
+  const { spanQuestion, id } = props;
+  const searchTextHighlight = useSearchTextHighlight(id);
   const spanAnnotationSupported = ref(true);
   const answer = spanQuestion.answer as SpanQuestionAnswer;
   const initialConfiguration = {

argilla-frontend/components/features/annotation/container/fields/text-field/TextField.vue

@@ -16,13 +16,13 @@
         </BaseButton>
       </BaseActionTooltip>
     </div>
-    <div :id="`fields-content-${name}`" class="content-area --body1">
+    <div :id="`fields-content-${id}`" class="content-area --body1">
       <MarkdownRenderer v-if="useMarkdown" :markdown="fieldText" />
       <Sandbox v-else-if="isHTML" :content="fieldText" />
       <div v-else :class="classes" v-html="fieldText" />
       <template>
-        <style :key="name" scoped>
-          ::highlight(search-text-highlight-{{name}}) {
+        <style :key="id" scoped>
+          ::highlight(search-text-highlight-{{id}}) {
             color: #ff675f;
           }
         </style>
@@ -36,7 +36,7 @@ import { useTextFieldViewModel } from "./useTextFieldViewModel";
 
 export default {
   props: {
-    name: {
+    id: {
       type: String,
       required: true,
     },

argilla-frontend/components/features/annotation/container/fields/text-field/useTextFieldViewModel.ts

@@ -2,10 +2,10 @@ import { onMounted, watch } from "vue-demi";
 import { useSearchTextHighlight } from "../useSearchTextHighlight";
 
 export const useTextFieldViewModel = (props: {
-  name: string;
+  id: string;
   searchText: string;
 }) => {
-  const { highlightText } = useSearchTextHighlight(props.name);
+  const { highlightText } = useSearchTextHighlight(props.id);
 
   watch(
     () => props.searchText,

argilla-frontend/dev.frontend.Dockerfile

@@ -1,6 +1,6 @@
-ARG ARGILLA_SERVER_TAG=main
+ARG ARGILLA_SERVER_TAG=develop
 
-FROM argilla/argilla-quickstart:${ARGILLA_SERVER_TAG}
+FROM argilladev/argilla-hf-spaces:${ARGILLA_SERVER_TAG}
 
 USER root
 

argilla-frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "argilla",
-  "version": "2.5.0dev0",
+  "version": "2.6.0dev0",
   "private": true,
   "scripts": {
     "dev": "nuxt",

argilla-server/src/argilla_server/_version.py

@@ -15,4 +15,4 @@
 # coding: utf-8
 #
 
-__version__ = "2.5.0dev0"
+__version__ = "2.6.0dev0"

argilla-server/src/argilla_server/alembic/versions/580a6553186f_add_datasets_users_table.py

@@ -0,0 +1,69 @@
+#  Copyright 2021-present, the Recognai S.L. team.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""add datasets_users table
+
+Revision ID: 580a6553186f
+Revises: 6ed1b8bf8e08
+Create Date: 2024-11-20 12:15:24.631417
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "580a6553186f"
+down_revision = "6ed1b8bf8e08"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "datasets_users",
+        sa.Column("dataset_id", sa.Uuid(), nullable=False),
+        sa.Column("user_id", sa.Uuid(), nullable=False),
+        sa.Column("inserted_at", sa.DateTime(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), nullable=False),
+        sa.ForeignKeyConstraint(["dataset_id"], ["datasets.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("dataset_id", "user_id"),
+    )
+    op.create_index(op.f("ix_datasets_users_dataset_id"), "datasets_users", ["dataset_id"], unique=False)
+    op.create_index(op.f("ix_datasets_users_user_id"), "datasets_users", ["user_id"], unique=False)
+
+    bind = op.get_bind()
+
+    statement = """
+        INSERT INTO datasets_users (dataset_id, user_id, inserted_at, updated_at)
+        SELECT dataset_id, user_id, {now_func}, {now_func} FROM (
+            SELECT DISTINCT records.dataset_id AS dataset_id, responses.user_id as user_id
+            FROM responses
+            JOIN records ON records.id = responses.record_id
+        ) AS subquery
+    """
+
+    if bind.dialect.name == "postgresql":
+        op.execute(statement.format(now_func="NOW()"))
+    elif bind.dialect.name == "sqlite":
+        op.execute(statement.format(now_func="datetime('now')"))
+    else:
+        raise Exception("Unsupported database dialect")
+
+
+def downgrade() -> None:
+    op.drop_index(op.f("ix_datasets_users_user_id"), table_name="datasets_users")
+    op.drop_index(op.f("ix_datasets_users_dataset_id"), table_name="datasets_users")
+    op.drop_table("datasets_users")

argilla-server/src/argilla_server/api/handlers/v1/datasets/datasets.py

@@ -155,12 +155,12 @@ async def get_current_user_dataset_metrics(
 
     await authorize(current_user, DatasetPolicy.get(dataset))
 
-    result = await datasets.get_user_dataset_metrics(search_engine, current_user, dataset)
+    result = await datasets.get_user_dataset_metrics(db, search_engine, current_user, dataset)
 
     return DatasetMetrics(responses=result)
 
 
-@router.get("/datasets/{dataset_id}/progress", response_model=DatasetProgress)
+@router.get("/datasets/{dataset_id}/progress", response_model=DatasetProgress, response_model_exclude_unset=True)
 async def get_dataset_progress(
     *,
     dataset_id: UUID,
@@ -172,7 +172,7 @@ async def get_dataset_progress(
 
     await authorize(current_user, DatasetPolicy.get(dataset))
 
-    result = await datasets.get_dataset_progress(search_engine, dataset)
+    result = await datasets.get_dataset_progress(db, search_engine, dataset)
 
     return DatasetProgress(**result)
 
@@ -182,14 +182,13 @@ async def get_dataset_users_progress(
     *,
     dataset_id: UUID,
     db: AsyncSession = Depends(get_async_db),
-    search_engine: SearchEngine = Depends(get_search_engine),
     current_user: User = Security(auth.get_current_user),
 ):
     dataset = await Dataset.get_or_raise(db, dataset_id)
 
     await authorize(current_user, DatasetPolicy.get(dataset))
 
-    progress = await datasets.get_dataset_users_progress(dataset.id)
+    progress = await datasets.get_dataset_users_progress(db, dataset)
 
     return UsersProgress(users=progress)
 

argilla-server/src/argilla_server/api/schemas/v1/datasets.py

@@ -84,12 +84,6 @@ class DatasetMetrics(BaseModel):
     responses: ResponseMetrics
 
 
-class DatasetProgress(BaseModel):
-    total: int
-    completed: int
-    pending: int
-
-
 class RecordResponseDistribution(BaseModel):
     submitted: int = 0
     discarded: int = 0
@@ -101,6 +95,15 @@ class UserProgress(BaseModel):
     completed: RecordResponseDistribution = RecordResponseDistribution()
     pending: RecordResponseDistribution = RecordResponseDistribution()
 
+    model_config = ConfigDict(from_attributes=True)
+
+
+class DatasetProgress(BaseModel):
+    total: int
+    completed: int
+    pending: int
+    users: List[UserProgress] = Field(default_factory=list)
+
 
 class UsersProgress(BaseModel):
     users: List[UserProgress]

argilla-server/src/argilla_server/api/schemas/v1/records.py

@@ -140,6 +140,8 @@ def validate_chat_field_content(cls, fields: Any):
                     fields[key] = [
                         item if isinstance(item, ChatFieldValue) else ChatFieldValue(**item) for item in value
                     ]
+                except TypeError as e:
+                    raise ValueError(f"Error parsing chat field '{key}': {e}")
                 except ValidationError as e:
                     raise ValueError(f"Error parsing chat field '{key}': {e.errors()}")
 

argilla-server/src/argilla_server/bulk/records_bulk.py

@@ -30,6 +30,7 @@
 )
 from argilla_server.api.schemas.v1.responses import UserResponseCreate
 from argilla_server.api.schemas.v1.suggestions import SuggestionCreate
+from argilla_server.models.database import DatasetUser
 from argilla_server.webhooks.v1.enums import RecordEvent
 from argilla_server.webhooks.v1.records import notify_record_event as notify_record_event_v1
 from argilla_server.contexts import distribution
@@ -109,13 +110,22 @@ async def _upsert_records_responses(
         self, records_and_responses: List[Tuple[Record, List[UserResponseCreate]]]
     ) -> List[Response]:
         upsert_many_responses = []
+        datasets_users = set()
         for idx, (record, responses) in enumerate(records_and_responses):
             for response_create in responses or []:
                 upsert_many_responses.append(dict(**response_create.model_dump(), record_id=record.id))
+                datasets_users.add((response_create.user_id, record.dataset_id))
 
         if not upsert_many_responses:
             return []
 
+        await DatasetUser.upsert_many(
+            self._db,
+            objects=[{"user_id": user_id, "dataset_id": dataset_id} for user_id, dataset_id in datasets_users],
+            constraints=[DatasetUser.user_id, DatasetUser.dataset_id],
+            autocommit=False,
+        )
+
         return await Response.upsert_many(
             self._db,
             objects=upsert_many_responses,