From 41cbcbc07f3999d6ced0c6e9f45d7642c3737b15 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Wed, 6 Nov 2024 22:15:35 -0500 Subject: [PATCH 01/26] fixing type on env-sample to .env-template --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index eeecb598d..99baf8111 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ docker compose -f docker-compose-dev.yaml up -d > Make sure you have Python 3.10 or 3.11 installed. 1. Export required environment variables or prepare a `.env` file in the project folder: - - Copy [.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) and create `.env`. + - Copy [.env-template](https://github.com/arc53/DocsGPT/blob/main/application/.env-template) and create `.env`. (check out [`application/core/settings.py`](application/core/settings.py) if you want to see more config options.) From bf0dd6946eb426034ff981a5e486b3ee6c9a501e Mon Sep 17 00:00:00 2001 From: fadingNA Date: Wed, 6 Nov 2024 22:16:00 -0500 Subject: [PATCH 02/26] add page, row_per_page to api/combine --- application/api/user/routes.py | 98 ++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 33 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 8e62683e6..6028dfa2f 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -74,7 +74,8 @@ def post(self): ) try: - conversations_collection.delete_one({"_id": ObjectId(conversation_id)}) + conversations_collection.delete_one( + {"_id": ObjectId(conversation_id)}) except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) return make_response(jsonify({"success": True}), 200) @@ -221,7 +222,8 @@ def get(self): ids = request.args.get("path") if not ids: return make_response( - jsonify({"success": False, "message": "Missing required fields"}), 400 + jsonify( + {"success": False, "message": "Missing required fields"}), 400 ) try: @@ -244,7 +246,8 @@ def get(self): source_id = request.args.get("source_id") if not source_id: return make_response( - jsonify({"success": False, "message": "Missing required fields"}), 400 + jsonify( + {"success": False, "message": "Missing required fields"}), 400 ) try: @@ -255,7 +258,8 @@ def get(self): return make_response(jsonify({"status": "not found"}), 404) if settings.VECTOR_STORE == "faiss": - shutil.rmtree(os.path.join(current_dir, "indexes", str(doc["_id"]))) + shutil.rmtree(os.path.join( + current_dir, "indexes", str(doc["_id"]))) else: vectorstore = VectorCreator.create_vectorstore( settings.VECTOR_STORE, source_id=str(doc["_id"]) @@ -305,7 +309,8 @@ def post(self): user = secure_filename(request.form["user"]) job_name = secure_filename(request.form["name"]) try: - save_dir = os.path.join(current_dir, settings.UPLOAD_FOLDER, user, job_name) + save_dir = os.path.join( + current_dir, settings.UPLOAD_FOLDER, user, job_name) os.makedirs(save_dir, exist_ok=True) if len(files) > 1: @@ -435,6 +440,8 @@ def get(self): user = "local" sort_field = request.args.get('sort', 'date') # Default to 'date' sort_order = request.args.get('order', "desc") # Default to 'desc' + page_number = request.args.get('page', 1) # Default to 1 + rows_per_page = request.args.get('rows', 10) # Default to 10 data = [ { "name": "default", @@ -447,7 +454,7 @@ def get(self): ] try: - for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order=="asc" else -1): + for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order == "asc" else -1): data.append( { "id": str(index["_id"]), @@ -485,10 +492,14 @@ def get(self): "retriever": "brave_search", } ) + + first_index = (int(page_number) - 1) * int(rows_per_page) + last_index = first_index + int(rows_per_page) + paginated_docs = data[first_index:last_index] except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) - return make_response(jsonify(data), 200) + return make_response(jsonify(paginated_data), 200) @user_ns.route("/api/docs_check") @@ -595,7 +606,8 @@ def get(self): try: if prompt_id == "default": with open( - os.path.join(current_dir, "prompts", "chat_combine_default.txt"), + os.path.join(current_dir, "prompts", + "chat_combine_default.txt"), "r", ) as f: chat_combine_template = f.read() @@ -603,7 +615,8 @@ def get(self): elif prompt_id == "creative": with open( - os.path.join(current_dir, "prompts", "chat_combine_creative.txt"), + os.path.join(current_dir, "prompts", + "chat_combine_creative.txt"), "r", ) as f: chat_reduce_creative = f.read() @@ -611,7 +624,8 @@ def get(self): elif prompt_id == "strict": with open( - os.path.join(current_dir, "prompts", "chat_combine_strict.txt"), "r" + os.path.join(current_dir, "prompts", + "chat_combine_strict.txt"), "r" ) as f: chat_reduce_strict = f.read() return make_response(jsonify({"content": chat_reduce_strict}), 200) @@ -627,7 +641,8 @@ def get(self): class DeletePrompt(Resource): delete_prompt_model = api.model( "DeletePromptModel", - {"id": fields.String(required=True, description="Prompt ID to delete")}, + {"id": fields.String( + required=True, description="Prompt ID to delete")}, ) @api.expect(delete_prompt_model) @@ -747,7 +762,8 @@ def post(self): "chunks": data["chunks"], } if "source" in data and ObjectId.is_valid(data["source"]): - new_api_key["source"] = DBRef("sources", ObjectId(data["source"])) + new_api_key["source"] = DBRef( + "sources", ObjectId(data["source"])) if "retriever" in data: new_api_key["retriever"] = data["retriever"] @@ -763,7 +779,8 @@ def post(self): class DeleteApiKey(Resource): delete_api_key_model = api.model( "DeleteApiKeyModel", - {"id": fields.String(required=True, description="API Key ID to delete")}, + {"id": fields.String( + required=True, description="API Key ID to delete")}, ) @api.expect(delete_api_key_model) @@ -776,7 +793,8 @@ def post(self): return missing_fields try: - result = api_key_collection.delete_one({"_id": ObjectId(data["id"])}) + result = api_key_collection.delete_one( + {"_id": ObjectId(data["id"])}) if result.deleted_count == 0: return {"success": False, "message": "API Key not found"}, 404 except Exception as err: @@ -811,7 +829,8 @@ def post(self): is_promptable = request.args.get("isPromptable", type=inputs.boolean) if is_promptable is None: return make_response( - jsonify({"success": False, "message": "isPromptable is required"}), 400 + jsonify( + {"success": False, "message": "isPromptable is required"}), 400 ) user = data.get("user", "local") @@ -975,7 +994,8 @@ def post(self): ) return make_response( jsonify( - {"success": True, "identifier": str(explicit_binary.as_uuid())} + {"success": True, "identifier": str( + explicit_binary.as_uuid())} ), 201, ) @@ -991,7 +1011,8 @@ def get(self, identifier: str): query_uuid = Binary.from_uuid( uuid.UUID(identifier), UuidRepresentation.STANDARD ) - shared = shared_conversations_collections.find_one({"uuid": query_uuid}) + shared = shared_conversations_collections.find_one( + {"uuid": query_uuid}) conversation_queries = [] if ( @@ -1071,7 +1092,8 @@ def post(self): try: api_key = ( - api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"] + api_key_collection.find_one( + {"_id": ObjectId(api_key_id)})["key"] if api_key_id else None ) @@ -1119,7 +1141,8 @@ def post(self): jsonify({"success": False, "message": "Invalid option"}), 400 ) start_date = end_date - datetime.timedelta(days=filter_days) - start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0) + start_date = start_date.replace( + hour=0, minute=0, second=0, microsecond=0) end_date = end_date.replace( hour=23, minute=59, second=59, microsecond=999999 ) @@ -1162,11 +1185,14 @@ def post(self): for entry in message_data: if filter_option == "last_hour": - daily_messages[entry["_id"]["minute"]] = entry["total_messages"] + daily_messages[entry["_id"]["minute"] + ] = entry["total_messages"] elif filter_option == "last_24_hour": - daily_messages[entry["_id"]["hour"]] = entry["total_messages"] + daily_messages[entry["_id"]["hour"] + ] = entry["total_messages"] else: - daily_messages[entry["_id"]["day"]] = entry["total_messages"] + daily_messages[entry["_id"]["day"] + ] = entry["total_messages"] except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) @@ -1206,7 +1232,8 @@ def post(self): try: api_key = ( - api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"] + api_key_collection.find_one( + {"_id": ObjectId(api_key_id)})["key"] if api_key_id else None ) @@ -1264,7 +1291,8 @@ def post(self): jsonify({"success": False, "message": "Invalid option"}), 400 ) start_date = end_date - datetime.timedelta(days=filter_days) - start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0) + start_date = start_date.replace( + hour=0, minute=0, second=0, microsecond=0) end_date = end_date.replace( hour=23, minute=59, second=59, microsecond=999999 ) @@ -1313,11 +1341,14 @@ def post(self): for entry in token_usage_data: if filter_option == "last_hour": - daily_token_usage[entry["_id"]["minute"]] = entry["total_tokens"] + daily_token_usage[entry["_id"] + ["minute"]] = entry["total_tokens"] elif filter_option == "last_24_hour": - daily_token_usage[entry["_id"]["hour"]] = entry["total_tokens"] + daily_token_usage[entry["_id"] + ["hour"]] = entry["total_tokens"] else: - daily_token_usage[entry["_id"]["day"]] = entry["total_tokens"] + daily_token_usage[entry["_id"] + ["day"]] = entry["total_tokens"] except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) @@ -1357,7 +1388,8 @@ def post(self): try: api_key = ( - api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"] + api_key_collection.find_one( + {"_id": ObjectId(api_key_id)})["key"] if api_key_id else None ) @@ -1459,7 +1491,8 @@ def post(self): jsonify({"success": False, "message": "Invalid option"}), 400 ) start_date = end_date - datetime.timedelta(days=filter_days) - start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0) + start_date = start_date.replace( + hour=0, minute=0, second=0, microsecond=0) end_date = end_date.replace( hour=23, minute=59, second=59, microsecond=999999 ) @@ -1575,7 +1608,8 @@ def post(self): try: api_key = ( - api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"] + api_key_collection.find_one( + {"_id": ObjectId(api_key_id)})["key"] if api_key_id else None ) @@ -1686,8 +1720,6 @@ def post(self): try: tts_instance = GoogleTTS() audio_base64, detected_language = tts_instance.text_to_speech(text) - return make_response(jsonify({"success": True,'audio_base64': audio_base64,'lang':detected_language}), 200) + return make_response(jsonify({"success": True, 'audio_base64': audio_base64, 'lang': detected_language}), 200) except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) - - From d68e731ffd374f26412fa345c71e2f823d086ee8 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Wed, 6 Nov 2024 22:19:16 -0500 Subject: [PATCH 03/26] revert format code --- application/api/user/routes.py | 98 ++++++++++++---------------------- 1 file changed, 33 insertions(+), 65 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 6028dfa2f..8e62683e6 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -74,8 +74,7 @@ def post(self): ) try: - conversations_collection.delete_one( - {"_id": ObjectId(conversation_id)}) + conversations_collection.delete_one({"_id": ObjectId(conversation_id)}) except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) return make_response(jsonify({"success": True}), 200) @@ -222,8 +221,7 @@ def get(self): ids = request.args.get("path") if not ids: return make_response( - jsonify( - {"success": False, "message": "Missing required fields"}), 400 + jsonify({"success": False, "message": "Missing required fields"}), 400 ) try: @@ -246,8 +244,7 @@ def get(self): source_id = request.args.get("source_id") if not source_id: return make_response( - jsonify( - {"success": False, "message": "Missing required fields"}), 400 + jsonify({"success": False, "message": "Missing required fields"}), 400 ) try: @@ -258,8 +255,7 @@ def get(self): return make_response(jsonify({"status": "not found"}), 404) if settings.VECTOR_STORE == "faiss": - shutil.rmtree(os.path.join( - current_dir, "indexes", str(doc["_id"]))) + shutil.rmtree(os.path.join(current_dir, "indexes", str(doc["_id"]))) else: vectorstore = VectorCreator.create_vectorstore( settings.VECTOR_STORE, source_id=str(doc["_id"]) @@ -309,8 +305,7 @@ def post(self): user = secure_filename(request.form["user"]) job_name = secure_filename(request.form["name"]) try: - save_dir = os.path.join( - current_dir, settings.UPLOAD_FOLDER, user, job_name) + save_dir = os.path.join(current_dir, settings.UPLOAD_FOLDER, user, job_name) os.makedirs(save_dir, exist_ok=True) if len(files) > 1: @@ -440,8 +435,6 @@ def get(self): user = "local" sort_field = request.args.get('sort', 'date') # Default to 'date' sort_order = request.args.get('order', "desc") # Default to 'desc' - page_number = request.args.get('page', 1) # Default to 1 - rows_per_page = request.args.get('rows', 10) # Default to 10 data = [ { "name": "default", @@ -454,7 +447,7 @@ def get(self): ] try: - for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order == "asc" else -1): + for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order=="asc" else -1): data.append( { "id": str(index["_id"]), @@ -492,14 +485,10 @@ def get(self): "retriever": "brave_search", } ) - - first_index = (int(page_number) - 1) * int(rows_per_page) - last_index = first_index + int(rows_per_page) - paginated_docs = data[first_index:last_index] except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) - return make_response(jsonify(paginated_data), 200) + return make_response(jsonify(data), 200) @user_ns.route("/api/docs_check") @@ -606,8 +595,7 @@ def get(self): try: if prompt_id == "default": with open( - os.path.join(current_dir, "prompts", - "chat_combine_default.txt"), + os.path.join(current_dir, "prompts", "chat_combine_default.txt"), "r", ) as f: chat_combine_template = f.read() @@ -615,8 +603,7 @@ def get(self): elif prompt_id == "creative": with open( - os.path.join(current_dir, "prompts", - "chat_combine_creative.txt"), + os.path.join(current_dir, "prompts", "chat_combine_creative.txt"), "r", ) as f: chat_reduce_creative = f.read() @@ -624,8 +611,7 @@ def get(self): elif prompt_id == "strict": with open( - os.path.join(current_dir, "prompts", - "chat_combine_strict.txt"), "r" + os.path.join(current_dir, "prompts", "chat_combine_strict.txt"), "r" ) as f: chat_reduce_strict = f.read() return make_response(jsonify({"content": chat_reduce_strict}), 200) @@ -641,8 +627,7 @@ def get(self): class DeletePrompt(Resource): delete_prompt_model = api.model( "DeletePromptModel", - {"id": fields.String( - required=True, description="Prompt ID to delete")}, + {"id": fields.String(required=True, description="Prompt ID to delete")}, ) @api.expect(delete_prompt_model) @@ -762,8 +747,7 @@ def post(self): "chunks": data["chunks"], } if "source" in data and ObjectId.is_valid(data["source"]): - new_api_key["source"] = DBRef( - "sources", ObjectId(data["source"])) + new_api_key["source"] = DBRef("sources", ObjectId(data["source"])) if "retriever" in data: new_api_key["retriever"] = data["retriever"] @@ -779,8 +763,7 @@ def post(self): class DeleteApiKey(Resource): delete_api_key_model = api.model( "DeleteApiKeyModel", - {"id": fields.String( - required=True, description="API Key ID to delete")}, + {"id": fields.String(required=True, description="API Key ID to delete")}, ) @api.expect(delete_api_key_model) @@ -793,8 +776,7 @@ def post(self): return missing_fields try: - result = api_key_collection.delete_one( - {"_id": ObjectId(data["id"])}) + result = api_key_collection.delete_one({"_id": ObjectId(data["id"])}) if result.deleted_count == 0: return {"success": False, "message": "API Key not found"}, 404 except Exception as err: @@ -829,8 +811,7 @@ def post(self): is_promptable = request.args.get("isPromptable", type=inputs.boolean) if is_promptable is None: return make_response( - jsonify( - {"success": False, "message": "isPromptable is required"}), 400 + jsonify({"success": False, "message": "isPromptable is required"}), 400 ) user = data.get("user", "local") @@ -994,8 +975,7 @@ def post(self): ) return make_response( jsonify( - {"success": True, "identifier": str( - explicit_binary.as_uuid())} + {"success": True, "identifier": str(explicit_binary.as_uuid())} ), 201, ) @@ -1011,8 +991,7 @@ def get(self, identifier: str): query_uuid = Binary.from_uuid( uuid.UUID(identifier), UuidRepresentation.STANDARD ) - shared = shared_conversations_collections.find_one( - {"uuid": query_uuid}) + shared = shared_conversations_collections.find_one({"uuid": query_uuid}) conversation_queries = [] if ( @@ -1092,8 +1071,7 @@ def post(self): try: api_key = ( - api_key_collection.find_one( - {"_id": ObjectId(api_key_id)})["key"] + api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"] if api_key_id else None ) @@ -1141,8 +1119,7 @@ def post(self): jsonify({"success": False, "message": "Invalid option"}), 400 ) start_date = end_date - datetime.timedelta(days=filter_days) - start_date = start_date.replace( - hour=0, minute=0, second=0, microsecond=0) + start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0) end_date = end_date.replace( hour=23, minute=59, second=59, microsecond=999999 ) @@ -1185,14 +1162,11 @@ def post(self): for entry in message_data: if filter_option == "last_hour": - daily_messages[entry["_id"]["minute"] - ] = entry["total_messages"] + daily_messages[entry["_id"]["minute"]] = entry["total_messages"] elif filter_option == "last_24_hour": - daily_messages[entry["_id"]["hour"] - ] = entry["total_messages"] + daily_messages[entry["_id"]["hour"]] = entry["total_messages"] else: - daily_messages[entry["_id"]["day"] - ] = entry["total_messages"] + daily_messages[entry["_id"]["day"]] = entry["total_messages"] except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) @@ -1232,8 +1206,7 @@ def post(self): try: api_key = ( - api_key_collection.find_one( - {"_id": ObjectId(api_key_id)})["key"] + api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"] if api_key_id else None ) @@ -1291,8 +1264,7 @@ def post(self): jsonify({"success": False, "message": "Invalid option"}), 400 ) start_date = end_date - datetime.timedelta(days=filter_days) - start_date = start_date.replace( - hour=0, minute=0, second=0, microsecond=0) + start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0) end_date = end_date.replace( hour=23, minute=59, second=59, microsecond=999999 ) @@ -1341,14 +1313,11 @@ def post(self): for entry in token_usage_data: if filter_option == "last_hour": - daily_token_usage[entry["_id"] - ["minute"]] = entry["total_tokens"] + daily_token_usage[entry["_id"]["minute"]] = entry["total_tokens"] elif filter_option == "last_24_hour": - daily_token_usage[entry["_id"] - ["hour"]] = entry["total_tokens"] + daily_token_usage[entry["_id"]["hour"]] = entry["total_tokens"] else: - daily_token_usage[entry["_id"] - ["day"]] = entry["total_tokens"] + daily_token_usage[entry["_id"]["day"]] = entry["total_tokens"] except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) @@ -1388,8 +1357,7 @@ def post(self): try: api_key = ( - api_key_collection.find_one( - {"_id": ObjectId(api_key_id)})["key"] + api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"] if api_key_id else None ) @@ -1491,8 +1459,7 @@ def post(self): jsonify({"success": False, "message": "Invalid option"}), 400 ) start_date = end_date - datetime.timedelta(days=filter_days) - start_date = start_date.replace( - hour=0, minute=0, second=0, microsecond=0) + start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0) end_date = end_date.replace( hour=23, minute=59, second=59, microsecond=999999 ) @@ -1608,8 +1575,7 @@ def post(self): try: api_key = ( - api_key_collection.find_one( - {"_id": ObjectId(api_key_id)})["key"] + api_key_collection.find_one({"_id": ObjectId(api_key_id)})["key"] if api_key_id else None ) @@ -1720,6 +1686,8 @@ def post(self): try: tts_instance = GoogleTTS() audio_base64, detected_language = tts_instance.text_to_speech(text) - return make_response(jsonify({"success": True, 'audio_base64': audio_base64, 'lang': detected_language}), 200) + return make_response(jsonify({"success": True,'audio_base64': audio_base64,'lang':detected_language}), 200) except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) + + From 101935ae46e0040e08e299f7d4c872d165f9a9a9 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Wed, 6 Nov 2024 22:20:25 -0500 Subject: [PATCH 04/26] add pagination with default code format --- application/api/user/routes.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 8e62683e6..7490d0fbe 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -435,6 +435,8 @@ def get(self): user = "local" sort_field = request.args.get('sort', 'date') # Default to 'date' sort_order = request.args.get('order', "desc") # Default to 'desc' + page = request.args.get('page', 1) # Default to 1 + rows_per_page = request.args.get('rows', 10) # Default to 10 data = [ { "name": "default", @@ -485,10 +487,15 @@ def get(self): "retriever": "brave_search", } ) + + first_index = (int(page_number) - 1) * int(rows_per_page) + last_index = first_index + int(rows_per_page) + paginated_docs = data[first_index:last_index] + except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) - return make_response(jsonify(data), 200) + return make_response(jsonify(paginated_docs), 200) @user_ns.route("/api/docs_check") From a35be6ae5712de134f31ea06271cd6d53242b299 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Wed, 6 Nov 2024 22:31:41 -0500 Subject: [PATCH 05/26] add arrow icons for pagination --- frontend/src/assets/double-arrow-left.svg | 5 +++++ frontend/src/assets/double-arrow-right.svg | 5 +++++ frontend/src/assets/single-left-arrow.svg | 3 +++ frontend/src/assets/single-right-arrow.svg | 3 +++ 4 files changed, 16 insertions(+) create mode 100644 frontend/src/assets/double-arrow-left.svg create mode 100644 frontend/src/assets/double-arrow-right.svg create mode 100644 frontend/src/assets/single-left-arrow.svg create mode 100644 frontend/src/assets/single-right-arrow.svg diff --git a/frontend/src/assets/double-arrow-left.svg b/frontend/src/assets/double-arrow-left.svg new file mode 100644 index 000000000..cab9ff900 --- /dev/null +++ b/frontend/src/assets/double-arrow-left.svg @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/frontend/src/assets/double-arrow-right.svg b/frontend/src/assets/double-arrow-right.svg new file mode 100644 index 000000000..0d5167c20 --- /dev/null +++ b/frontend/src/assets/double-arrow-right.svg @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/frontend/src/assets/single-left-arrow.svg b/frontend/src/assets/single-left-arrow.svg new file mode 100644 index 000000000..f28b25921 --- /dev/null +++ b/frontend/src/assets/single-left-arrow.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/frontend/src/assets/single-right-arrow.svg b/frontend/src/assets/single-right-arrow.svg new file mode 100644 index 000000000..85729e570 --- /dev/null +++ b/frontend/src/assets/single-right-arrow.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file From 5dae074c95e1e9f8742fbf41a6e991fbec2db6af Mon Sep 17 00:00:00 2001 From: fadingNA Date: Wed, 6 Nov 2024 22:31:56 -0500 Subject: [PATCH 06/26] pagination component --- .../src/components/DocumentPagination.tsx | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 frontend/src/components/DocumentPagination.tsx diff --git a/frontend/src/components/DocumentPagination.tsx b/frontend/src/components/DocumentPagination.tsx new file mode 100644 index 000000000..b5d82456b --- /dev/null +++ b/frontend/src/components/DocumentPagination.tsx @@ -0,0 +1,115 @@ +import React, { useState } from 'react'; +import SingleArrowLeft from '../assets/single-left-arrow.svg'; +import SingleArrowRight from '../assets/single-right-arrow.svg'; +import DoubleArrowLeft from '../assets/double-arrow-left.svg'; +import DoubleArrowRight from '../assets/double-arrow-right.svg'; + +interface PaginationProps { + currentPage: number; + totalPages: number; + rowsPerPage: number; + onPageChange: (page: number) => void; + onRowsPerPageChange: (rows: number) => void; +} + +const Pagination: React.FC = ({ + currentPage, + totalPages, + rowsPerPage, + onPageChange, + onRowsPerPageChange, +}) => { + const [rowsPerPageOptions] = useState([5, 10, 15, 20]); + + const handlePreviousPage = () => { + if (currentPage > 1) { + onPageChange(currentPage - 1); + } + }; + + const handleNextPage = () => { + if (currentPage < totalPages) { + onPageChange(currentPage + 1); + } + }; + + const handleFirstPage = () => { + onPageChange(1); + }; + + const handleLastPage = () => { + onPageChange(totalPages); + }; + + return ( +
+
+ Rows per page: + +
+ +
+ Page {currentPage} of {totalPages} +
+ +
+ + + + +
+
+ ); +}; + +export default Pagination; From ec3407df7ebd7380136e597a750d792eebf11b13 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Wed, 6 Nov 2024 22:32:37 -0500 Subject: [PATCH 07/26] add 2 fiels for make request to api/combine --- frontend/src/api/services/userService.ts | 11 +++++++++-- frontend/src/preferences/preferenceApi.ts | 9 ++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts index 53b38f50c..d55cc5bd1 100644 --- a/frontend/src/api/services/userService.ts +++ b/frontend/src/api/services/userService.ts @@ -2,8 +2,15 @@ import apiClient from '../client'; import endpoints from '../endpoints'; const userService = { - getDocs: (sort = 'date', order = 'desc'): Promise => - apiClient.get(`${endpoints.USER.DOCS}?sort=${sort}&order=${order}`), + getDocs: ( + sort = 'date', + order = 'desc', + pageNumber = 1, + rowsPerPage = 10, + ): Promise => + apiClient.get( + `${endpoints.USER.DOCS}?sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`, + ), checkDocs: (data: any): Promise => apiClient.post(endpoints.USER.DOCS_CHECK, data), getAPIKeys: (): Promise => apiClient.get(endpoints.USER.API_KEYS), diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts index af1060b83..226af119a 100644 --- a/frontend/src/preferences/preferenceApi.ts +++ b/frontend/src/preferences/preferenceApi.ts @@ -6,9 +6,16 @@ import { Doc } from '../models/misc'; export async function getDocs( sort = 'date', order = 'desc', + pageNumber = 1, + rowsPerPage = 5, ): Promise { try { - const response = await userService.getDocs(sort, order); + const response = await userService.getDocs( + sort, + order, + pageNumber, + rowsPerPage, + ); const data = await response.json(); const docs: Doc[] = []; From df2f69e85fab05e02b01ce6657d0ed43af756564 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Wed, 6 Nov 2024 22:33:26 -0500 Subject: [PATCH 08/26] adjust table of document with mx auto, add pagination to Documents Component --- frontend/src/index.css | 4 +- frontend/src/settings/Documents.tsx | 65 ++++++++++++++++++++++++----- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/frontend/src/index.css b/frontend/src/index.css index 4319403e7..090a31ca6 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -50,11 +50,11 @@ body.dark { @layer components { .table-default { - @apply block w-max table-auto content-center justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray; + @apply block w-max mx-auto table-fixed content-center justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray; } .table-default th { - @apply p-4 w-[244px] font-normal text-gray-400; /* Remove border-r */ + @apply p-4 w-[250px] font-normal text-gray-400; /* Remove border-r */ } .table-default th:last-child { diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx index f94d1a87b..cbb1048c3 100644 --- a/frontend/src/settings/Documents.tsx +++ b/frontend/src/settings/Documents.tsx @@ -1,4 +1,4 @@ -import React, { useState } from 'react'; +import React, { useState, useEffect } from 'react'; import PropTypes from 'prop-types'; import { useTranslation } from 'react-i18next'; import { useDispatch } from 'react-redux'; @@ -14,6 +14,7 @@ import { getDocs } from '../preferences/preferenceApi'; import { setSourceDocs } from '../preferences/preferenceSlice'; import Input from '../components/Input'; import Upload from '../upload/Upload'; // Import the Upload component +import Pagination from '../components/DocumentPagination'; // Utility function to format numbers const formatTokens = (tokens: number): string => { @@ -47,22 +48,55 @@ const Documents: React.FC = ({ const [loading, setLoading] = useState(false); const [sortField, setSortField] = useState<'date' | 'tokens'>('date'); const [sortOrder, setSortOrder] = useState<'asc' | 'desc'>('desc'); + // Pagination + const [currentPage, setCurrentPage] = useState(1); + const [rowsPerPage, setRowsPerPage] = useState(10); + const [totalPages, setTotalPages] = useState(0); + const [fetchedDocuments, setFetchedDocuments] = useState([]); + // Filter documents based on the search term + const filteredDocuments = documents?.filter((document) => + document.name.toLowerCase().includes(searchTerm.toLowerCase()), + ); + // State for documents + const startIndex = (currentPage - 1) * rowsPerPage; + const endIndex = startIndex + rowsPerPage; + const currentDocuments = filteredDocuments + ? filteredDocuments.slice(startIndex, endIndex) + : []; + const syncOptions = [ { label: 'Never', value: 'never' }, { label: 'Daily', value: 'daily' }, { label: 'Weekly', value: 'weekly' }, { label: 'Monthly', value: 'monthly' }, ]; - const refreshDocs = (field: 'date' | 'tokens') => { + + useEffect(() => { + if (fetchedDocuments && rowsPerPage > 0) { + setTotalPages( + Math.max(1, Math.ceil(fetchedDocuments.length / rowsPerPage)), + ); + } else { + setTotalPages(0); + } + }, [fetchedDocuments, rowsPerPage]); + + const refreshDocs = ( + field: 'date' | 'tokens', + pageNumber = currentPage, + rows = rowsPerPage, + ) => { if (field === sortField) { setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc'); } else { setSortOrder('desc'); setSortField(field); } - getDocs(sortField, sortOrder) + console.log('refreshDocs', field, sortOrder, pageNumber, rows); + getDocs(sortField, sortOrder, pageNumber, rows) .then((data) => { dispatch(setSourceDocs(data)); + setFetchedDocuments(data ? data : []); }) .catch((error) => console.error(error)) .finally(() => { @@ -85,11 +119,6 @@ const Documents: React.FC = ({ }); }; - // Filter documents based on the search term - const filteredDocuments = documents?.filter((document) => - document.name.toLowerCase().includes(searchTerm.toLowerCase()), - ); - return (
@@ -154,15 +183,15 @@ const Documents: React.FC = ({ - {!filteredDocuments?.length && ( + {!currentDocuments?.length && ( {t('settings.documents.noData')} )} - {filteredDocuments && - filteredDocuments.map((document, index) => ( + {Array.isArray(currentDocuments) && + currentDocuments.map((document, index) => ( {document.name} {document.date} @@ -221,6 +250,20 @@ const Documents: React.FC = ({
)}
+ { + setCurrentPage(page); + refreshDocs(sortField, page, rowsPerPage); + }} + onRowsPerPageChange={(rows) => { + setRowsPerPage(rows); + setCurrentPage(1); + refreshDocs(sortField, 1, rows); + }} + /> ); }; From 928303f27b07920aa93c2e7cb46e1ef1401f9d2f Mon Sep 17 00:00:00 2001 From: fadingNA Date: Wed, 6 Nov 2024 22:36:59 -0500 Subject: [PATCH 09/26] fix small bug name of page --- application/api/user/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 7490d0fbe..5295c2ee7 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -488,7 +488,7 @@ def get(self): } ) - first_index = (int(page_number) - 1) * int(rows_per_page) + first_index = (int(page) - 1) * int(rows_per_page) last_index = first_index + int(rows_per_page) paginated_docs = data[first_index:last_index] From 2a68cc998962a49a6a675f44bdc27adb098f4f54 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Thu, 7 Nov 2024 23:21:06 -0500 Subject: [PATCH 10/26] calculate totalPage, totalDoc(futreUse) --- application/api/user/routes.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 5295c2ee7..e5ad68aef 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -2,6 +2,7 @@ import os import shutil import uuid +import math from bson.binary import Binary, UuidRepresentation from bson.dbref import DBRef @@ -435,8 +436,8 @@ def get(self): user = "local" sort_field = request.args.get('sort', 'date') # Default to 'date' sort_order = request.args.get('order', "desc") # Default to 'desc' - page = request.args.get('page', 1) # Default to 1 - rows_per_page = request.args.get('rows', 10) # Default to 10 + page = int(request.args.get('page', 1)) # Default to 1 + rows_per_page = int(request.args.get('rows', 10)) # Default to 10 data = [ { "name": "default", @@ -487,15 +488,23 @@ def get(self): "retriever": "brave_search", } ) + total_documents = len(data) + total_pages = max(1, math.ceil(total_documents / rows_per_page)) - first_index = (int(page) - 1) * int(rows_per_page) - last_index = first_index + int(rows_per_page) + first_index = (page - 1) * rows_per_page + last_index = first_index + rows_per_page paginated_docs = data[first_index:last_index] except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) - - return make_response(jsonify(paginated_docs), 200) + + response = { + "paginated_docs": paginated_docs, + "totalDocuments": total_documents, + "totalPages": total_pages, + "documents":data + } + return make_response(jsonify(response), 200) @user_ns.route("/api/docs_check") From 7ff86a2aeee48149ce3aaa388de99e12aab0a0fa Mon Sep 17 00:00:00 2001 From: fadingNA Date: Thu, 7 Nov 2024 23:21:26 -0500 Subject: [PATCH 11/26] remove default value since provide on getDocs --- frontend/src/api/services/userService.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts index d55cc5bd1..bebadca50 100644 --- a/frontend/src/api/services/userService.ts +++ b/frontend/src/api/services/userService.ts @@ -3,10 +3,10 @@ import endpoints from '../endpoints'; const userService = { getDocs: ( - sort = 'date', - order = 'desc', - pageNumber = 1, - rowsPerPage = 10, + sort: string, + order: string, + pageNumber: number, + rowsPerPage: number, ): Promise => apiClient.get( `${endpoints.USER.DOCS}?sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`, From 3d03826db517b84a374447861c76ed81e024f1d6 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Thu, 7 Nov 2024 23:22:03 -0500 Subject: [PATCH 12/26] add overload function to gettotalpage, and new type --- frontend/src/models/misc.ts | 6 ++++ frontend/src/preferences/preferenceApi.ts | 40 ++++++++++++++++++----- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/frontend/src/models/misc.ts b/frontend/src/models/misc.ts index 9affd0ab4..5478722c5 100644 --- a/frontend/src/models/misc.ts +++ b/frontend/src/models/misc.ts @@ -14,6 +14,12 @@ export type Doc = { syncFrequency?: string; }; +export type GetDocsResponse = { + docs: Doc[]; + totalDocuments: number; + totalPages: number; +}; + export type PromptProps = { prompts: { name: string; id: string; type: string }[]; selectedPrompt: { name: string; id: string; type: string }; diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts index 226af119a..e77c9e3b2 100644 --- a/frontend/src/preferences/preferenceApi.ts +++ b/frontend/src/preferences/preferenceApi.ts @@ -1,14 +1,23 @@ import conversationService from '../api/services/conversationService'; import userService from '../api/services/userService'; -import { Doc } from '../models/misc'; +import { Doc, GetDocsResponse } from '../models/misc'; //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later. +export async function getDocs( + sort?: string, + order?: string, + pageNumber?: number, + rowsPerPage?: number, + withPagination?: true, +): Promise; + export async function getDocs( sort = 'date', order = 'desc', pageNumber = 1, rowsPerPage = 5, -): Promise { + withPagination = false, +): Promise { try { const response = await userService.getDocs( sort, @@ -17,14 +26,27 @@ export async function getDocs( rowsPerPage, ); const data = await response.json(); + console.log(data); + if (withPagination) { + const docs: Doc[] = []; + Array.isArray(data.paginated_docs) && + data.paginated_docs.forEach((doc: object) => { + docs.push(doc as Doc); + }); - const docs: Doc[] = []; - - data.forEach((doc: object) => { - docs.push(doc as Doc); - }); - - return docs; + const totalDocuments = data.totalDocuments || 0; + const totalPages = data.totalPages || 0; + console.log(`totalDocuments: ${totalDocuments}`); + console.log(`totalPages: ${totalPages}`); + return { docs, totalDocuments, totalPages }; + } else { + const docs: Doc[] = []; + Array.isArray(data.documents) && + data.documents.forEach((doc: object) => { + docs.push(doc as Doc); + }); + return docs; + } } catch (error) { console.log(error); return null; From a2967afb551956f726621d87afc6069eefb0d43c Mon Sep 17 00:00:00 2001 From: fadingNA Date: Thu, 7 Nov 2024 23:22:28 -0500 Subject: [PATCH 13/26] adjust Setting/Document to use pagination data from backend --- frontend/src/settings/Documents.tsx | 60 ++++++++++++++--------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx index cbb1048c3..819a4638e 100644 --- a/frontend/src/settings/Documents.tsx +++ b/frontend/src/settings/Documents.tsx @@ -51,18 +51,15 @@ const Documents: React.FC = ({ // Pagination const [currentPage, setCurrentPage] = useState(1); const [rowsPerPage, setRowsPerPage] = useState(10); - const [totalPages, setTotalPages] = useState(0); + const [totalPages, setTotalPages] = useState(1); + const [totalDocuments, setTotalDocuments] = useState(0); const [fetchedDocuments, setFetchedDocuments] = useState([]); // Filter documents based on the search term - const filteredDocuments = documents?.filter((document) => + const filteredDocuments = fetchedDocuments?.filter((document) => document.name.toLowerCase().includes(searchTerm.toLowerCase()), ); // State for documents - const startIndex = (currentPage - 1) * rowsPerPage; - const endIndex = startIndex + rowsPerPage; - const currentDocuments = filteredDocuments - ? filteredDocuments.slice(startIndex, endIndex) - : []; + const currentDocuments = filteredDocuments ?? []; const syncOptions = [ { label: 'Never', value: 'never' }, @@ -71,32 +68,31 @@ const Documents: React.FC = ({ { label: 'Monthly', value: 'monthly' }, ]; - useEffect(() => { - if (fetchedDocuments && rowsPerPage > 0) { - setTotalPages( - Math.max(1, Math.ceil(fetchedDocuments.length / rowsPerPage)), - ); - } else { - setTotalPages(0); - } - }, [fetchedDocuments, rowsPerPage]); - const refreshDocs = ( field: 'date' | 'tokens', - pageNumber = currentPage, - rows = rowsPerPage, + pageNumber?: number, + rows?: number, ) => { - if (field === sortField) { - setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc'); - } else { - setSortOrder('desc'); - setSortField(field); + console.log(`field: ${field}, pageNumber: ${pageNumber}, rows: ${rows}`); + const page = pageNumber ?? currentPage; + const rowsPerPg = rows ?? rowsPerPage; + if (field !== undefined) { + if (field === sortField) { + // Toggle sort order + setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc'); + } else { + // Change sort field and reset order to 'desc' + setSortField(field); + setSortOrder('desc'); + } } - console.log('refreshDocs', field, sortOrder, pageNumber, rows); - getDocs(sortField, sortOrder, pageNumber, rows) + getDocs(sortField, sortOrder, page, rowsPerPg, true) .then((data) => { - dispatch(setSourceDocs(data)); - setFetchedDocuments(data ? data : []); + console.log(data); + dispatch(setSourceDocs(data ? data.docs : [])); + setFetchedDocuments(data ? data.docs : []); + setTotalPages(data ? data.totalPages : 0); + setTotalDocuments(data ? data.totalDocuments : 0); }) .catch((error) => console.error(error)) .finally(() => { @@ -118,7 +114,9 @@ const Documents: React.FC = ({ setLoading(false); }); }; - + useEffect(() => { + refreshDocs(sortField, currentPage, rowsPerPage); + }, []); return (
@@ -256,12 +254,12 @@ const Documents: React.FC = ({ rowsPerPage={rowsPerPage} onPageChange={(page) => { setCurrentPage(page); - refreshDocs(sortField, page, rowsPerPage); + refreshDocs(undefined, page, rowsPerPage); }} onRowsPerPageChange={(rows) => { setRowsPerPage(rows); setCurrentPage(1); - refreshDocs(sortField, 1, rows); + refreshDocs(undefined, 1, rows); }} />
From 4429755c096fb1b02113aa3917a6db7ef421d863 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Thu, 7 Nov 2024 23:23:01 -0500 Subject: [PATCH 14/26] add paramter type prevent warning lint --- frontend/src/settings/Documents.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx index 819a4638e..fc21a8f13 100644 --- a/frontend/src/settings/Documents.tsx +++ b/frontend/src/settings/Documents.tsx @@ -69,7 +69,7 @@ const Documents: React.FC = ({ ]; const refreshDocs = ( - field: 'date' | 'tokens', + field: 'date' | 'tokens' | undefined, pageNumber?: number, rows?: number, ) => { From dd9589b37a2f78b7eedf62fe356193fcb3054278 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Fri, 8 Nov 2024 11:02:13 -0500 Subject: [PATCH 15/26] fixing type error --- frontend/src/Navigation.tsx | 5 ++++- frontend/src/hooks/useDefaultDocument.ts | 11 +++++----- frontend/src/models/misc.ts | 1 - frontend/src/settings/Documents.tsx | 12 ++++------- frontend/src/settings/index.tsx | 7 +----- frontend/src/upload/Upload.tsx | 27 ++++++++++++++++-------- 6 files changed, 33 insertions(+), 30 deletions(-) diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx index 3591469b1..72c0d57a5 100644 --- a/frontend/src/Navigation.tsx +++ b/frontend/src/Navigation.tsx @@ -145,7 +145,10 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { dispatch(setSourceDocs(updatedDocs)); dispatch( setSelectedDocs( - updatedDocs?.find((doc) => doc.name.toLowerCase() === 'default'), + Array.isArray(updatedDocs) && + updatedDocs?.find( + (doc: Doc) => doc.name.toLowerCase() === 'default', + ), ), ); }) diff --git a/frontend/src/hooks/useDefaultDocument.ts b/frontend/src/hooks/useDefaultDocument.ts index 37374ce0b..7f4b98126 100644 --- a/frontend/src/hooks/useDefaultDocument.ts +++ b/frontend/src/hooks/useDefaultDocument.ts @@ -17,11 +17,12 @@ export default function useDefaultDocument() { getDocs().then((data) => { dispatch(setSourceDocs(data)); if (!selectedDoc) - data?.forEach((doc: Doc) => { - if (doc.model && doc.name === 'default') { - dispatch(setSelectedDocs(doc)); - } - }); + Array.isArray(data) && + data?.forEach((doc: Doc) => { + if (doc.model && doc.name === 'default') { + dispatch(setSelectedDocs(doc)); + } + }); }); }; diff --git a/frontend/src/models/misc.ts b/frontend/src/models/misc.ts index 5478722c5..3d5d908c5 100644 --- a/frontend/src/models/misc.ts +++ b/frontend/src/models/misc.ts @@ -28,7 +28,6 @@ export type PromptProps = { }; export type DocumentsProps = { - documents: Doc[] | null; handleDeleteDocument: (index: number, document: Doc) => void; }; diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx index fc21a8f13..ed7f2d890 100644 --- a/frontend/src/settings/Documents.tsx +++ b/frontend/src/settings/Documents.tsx @@ -33,13 +33,9 @@ const formatTokens = (tokens: number): string => { } }; -const Documents: React.FC = ({ - documents, - handleDeleteDocument, -}) => { +const Documents: React.FC = ({ handleDeleteDocument }) => { const { t } = useTranslation(); const dispatch = useDispatch(); - // State for search input const [searchTerm, setSearchTerm] = useState(''); // State for modal: active/inactive @@ -73,7 +69,6 @@ const Documents: React.FC = ({ pageNumber?: number, rows?: number, ) => { - console.log(`field: ${field}, pageNumber: ${pageNumber}, rows: ${rows}`); const page = pageNumber ?? currentPage; const rowsPerPg = rows ?? rowsPerPage; if (field !== undefined) { @@ -88,7 +83,6 @@ const Documents: React.FC = ({ } getDocs(sortField, sortOrder, page, rowsPerPg, true) .then((data) => { - console.log(data); dispatch(setSourceDocs(data ? data.docs : [])); setFetchedDocuments(data ? data.docs : []); setTotalPages(data ? data.totalPages : 0); @@ -114,9 +108,11 @@ const Documents: React.FC = ({ setLoading(false); }); }; + useEffect(() => { refreshDocs(sortField, currentPage, rowsPerPage); }, []); + return (
@@ -267,7 +263,7 @@ const Documents: React.FC = ({ }; Documents.propTypes = { - documents: PropTypes.array.isRequired, + //documents: PropTypes.array.isRequired, handleDeleteDocument: PropTypes.func.isRequired, }; diff --git a/frontend/src/settings/index.tsx b/frontend/src/settings/index.tsx index ea3d44281..24ea26812 100644 --- a/frontend/src/settings/index.tsx +++ b/frontend/src/settings/index.tsx @@ -70,12 +70,7 @@ export default function Settings() { case t('settings.general.label'): return ; case t('settings.documents.label'): - return ( - - ); + return ; case 'Widgets': return ( d.type?.toLowerCase() === 'local'), + Array.isArray(data) && + data?.find( + (d: Doc) => d.type?.toLowerCase() === 'local', + ), ), ); }); @@ -182,15 +185,21 @@ function Upload({ getDocs().then((data) => { dispatch(setSourceDocs(data)); const docIds = new Set( - sourceDocs?.map((doc: Doc) => (doc.id ? doc.id : null)), + (Array.isArray(sourceDocs) && + sourceDocs?.map((doc: Doc) => + doc.id ? doc.id : null, + )) || + [], ); - data?.map((updatedDoc: Doc) => { - if (updatedDoc.id && !docIds.has(updatedDoc.id)) { - //select the doc not present in the intersection of current Docs and fetched data - dispatch(setSelectedDocs(updatedDoc)); - return; - } - }); + if (data && Array.isArray(data.docs)) { + data.docs.map((updatedDoc: Doc) => { + if (updatedDoc.id && !docIds.has(updatedDoc.id)) { + // Select the doc not present in the intersection of current Docs and fetched data + dispatch(setSelectedDocs(updatedDoc)); + return; + } + }); + } }); setProgress( (progress) => From 5debb482651b11e77d3ecf81696b5aedf43cd00b Mon Sep 17 00:00:00 2001 From: fadingNA Date: Fri, 8 Nov 2024 11:02:13 -0500 Subject: [PATCH 16/26] Paginated With MongoDB / Create New Endpoint change routes /combine name, add route /api/source/paginated add new endpoint source/paginated fixing table responsive create new function to handling api/source/paginated --- application/api/user/routes.py | 96 +++++++++++++++++------ frontend/src/Navigation.tsx | 5 +- frontend/src/api/endpoints.ts | 3 +- frontend/src/api/services/userService.ts | 13 +-- frontend/src/hooks/useDefaultDocument.ts | 11 +-- frontend/src/index.css | 4 +- frontend/src/models/misc.ts | 2 +- frontend/src/preferences/preferenceApi.ts | 78 +++++++++--------- frontend/src/settings/Documents.tsx | 40 ++++++---- frontend/src/settings/index.tsx | 7 +- frontend/src/upload/Upload.tsx | 27 ++++--- 11 files changed, 174 insertions(+), 112 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index e5ad68aef..7d44ce9bf 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -7,7 +7,7 @@ from bson.binary import Binary, UuidRepresentation from bson.dbref import DBRef from bson.objectid import ObjectId -from flask import Blueprint, jsonify, make_response, request +from flask import Blueprint, jsonify, make_response, request, redirect from flask_restx import inputs, fields, Namespace, Resource from werkzeug.utils import secure_filename @@ -430,14 +430,63 @@ def get(self): @user_ns.route("/api/combine") +class RedirectToSources(Resource): + @api.doc( + description="Redirects /api/combine to /api/sources for backward compatibility" + ) + def get(self): + return redirect("/api/sources", code=301) + + +@user_ns.route("/api/sources/paginated") +class PaginatedSources(Resource): + @api.doc(description="Get document with pagination, sorting and filtering") + def get(self): + user = "local" + sort_field = request.args.get("sort", "date") # Default to 'date' + sort_order = request.args.get("order", "desc") # Default to 'desc' + page = int(request.args.get("page", 1)) # Default to 1 + rows_per_page = int(request.args.get("rows", 10)) # Default to 10 + + # Prepare + query = {"user": user} + total_documents = sources_collection.count_documents(query) + total_pages = max(1, math.ceil(total_documents / rows_per_page)) + sort_order = 1 if sort_order == "asc" else -1 + skip = (page - 1) * rows_per_page + + try: + documents = ( + sources_collection.find(query) + .sort(sort_field, sort_order) + .skip(skip) + .limit(rows_per_page) + ) + + paginated_docs = [] + for doc in documents: + doc["_id"] = str(doc["_id"]) + paginated_docs.append(doc) + + response = { + "total": total_documents, + "totalPages": total_pages, + "currentPage": page, + "paginated": paginated_docs, + } + return make_response(jsonify(response), 200) + + except Exception as err: + return make_response(jsonify({"success": False, "error": str(err)}), 400) + + +@user_ns.route("/api/sources") class CombinedJson(Resource): @api.doc(description="Provide JSON file with combined available indexes") def get(self): user = "local" - sort_field = request.args.get('sort', 'date') # Default to 'date' - sort_order = request.args.get('order', "desc") # Default to 'desc' - page = int(request.args.get('page', 1)) # Default to 1 - rows_per_page = int(request.args.get('rows', 10)) # Default to 10 + sort_field = request.args.get("sort", "date") # Default to 'date' + sort_order = request.args.get("order", "desc") # Default to 'desc' data = [ { "name": "default", @@ -450,7 +499,9 @@ def get(self): ] try: - for index in sources_collection.find({"user": user}).sort(sort_field, 1 if sort_order=="asc" else -1): + for index in sources_collection.find({"user": user}).sort( + sort_field, 1 if sort_order == "asc" else -1 + ): data.append( { "id": str(index["_id"]), @@ -488,23 +539,11 @@ def get(self): "retriever": "brave_search", } ) - total_documents = len(data) - total_pages = max(1, math.ceil(total_documents / rows_per_page)) - - first_index = (page - 1) * rows_per_page - last_index = first_index + rows_per_page - paginated_docs = data[first_index:last_index] except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) - - response = { - "paginated_docs": paginated_docs, - "totalDocuments": total_documents, - "totalPages": total_pages, - "documents":data - } - return make_response(jsonify(response), 200) + + return make_response(jsonify(data), 200) @user_ns.route("/api/docs_check") @@ -1690,7 +1729,9 @@ class TextToSpeech(Resource): tts_model = api.model( "TextToSpeechModel", { - "text": fields.String(required=True, description="Text to be synthesized as audio"), + "text": fields.String( + required=True, description="Text to be synthesized as audio" + ), }, ) @@ -1702,8 +1743,15 @@ def post(self): try: tts_instance = GoogleTTS() audio_base64, detected_language = tts_instance.text_to_speech(text) - return make_response(jsonify({"success": True,'audio_base64': audio_base64,'lang':detected_language}), 200) + return make_response( + jsonify( + { + "success": True, + "audio_base64": audio_base64, + "lang": detected_language, + } + ), + 200, + ) except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) - - diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx index 3591469b1..72c0d57a5 100644 --- a/frontend/src/Navigation.tsx +++ b/frontend/src/Navigation.tsx @@ -145,7 +145,10 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { dispatch(setSourceDocs(updatedDocs)); dispatch( setSelectedDocs( - updatedDocs?.find((doc) => doc.name.toLowerCase() === 'default'), + Array.isArray(updatedDocs) && + updatedDocs?.find( + (doc: Doc) => doc.name.toLowerCase() === 'default', + ), ), ); }) diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts index 84674049e..4e7112d04 100644 --- a/frontend/src/api/endpoints.ts +++ b/frontend/src/api/endpoints.ts @@ -1,7 +1,8 @@ const endpoints = { USER: { - DOCS: '/api/combine', + DOCS: '/api/sources', DOCS_CHECK: '/api/docs_check', + DOCS_PAGINATED: '/api/sources/paginated', API_KEYS: '/api/get_api_keys', CREATE_API_KEY: '/api/create_api_key', DELETE_API_KEY: '/api/delete_api_key', diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts index bebadca50..82f322449 100644 --- a/frontend/src/api/services/userService.ts +++ b/frontend/src/api/services/userService.ts @@ -2,15 +2,10 @@ import apiClient from '../client'; import endpoints from '../endpoints'; const userService = { - getDocs: ( - sort: string, - order: string, - pageNumber: number, - rowsPerPage: number, - ): Promise => - apiClient.get( - `${endpoints.USER.DOCS}?sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`, - ), + getDocs: (sort: string, order: string): Promise => + apiClient.get(`${endpoints.USER.DOCS}?sort=${sort}&order=${order}`), + getDocsWithPagination: (query: string): Promise => + apiClient.get(`${endpoints.USER.DOCS_PAGINATED}?${query}`), checkDocs: (data: any): Promise => apiClient.post(endpoints.USER.DOCS_CHECK, data), getAPIKeys: (): Promise => apiClient.get(endpoints.USER.API_KEYS), diff --git a/frontend/src/hooks/useDefaultDocument.ts b/frontend/src/hooks/useDefaultDocument.ts index 37374ce0b..7f4b98126 100644 --- a/frontend/src/hooks/useDefaultDocument.ts +++ b/frontend/src/hooks/useDefaultDocument.ts @@ -17,11 +17,12 @@ export default function useDefaultDocument() { getDocs().then((data) => { dispatch(setSourceDocs(data)); if (!selectedDoc) - data?.forEach((doc: Doc) => { - if (doc.model && doc.name === 'default') { - dispatch(setSelectedDocs(doc)); - } - }); + Array.isArray(data) && + data?.forEach((doc: Doc) => { + if (doc.model && doc.name === 'default') { + dispatch(setSelectedDocs(doc)); + } + }); }); }; diff --git a/frontend/src/index.css b/frontend/src/index.css index 090a31ca6..e8e9c9bdc 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -50,11 +50,11 @@ body.dark { @layer components { .table-default { - @apply block w-max mx-auto table-fixed content-center justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray; + @apply block w-full mx-auto table-fixed content-start justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray; } .table-default th { - @apply p-4 w-[250px] font-normal text-gray-400; /* Remove border-r */ + @apply p-4 w-1/4 font-normal text-gray-400 text-nowrap; /* Remove border-r */ } .table-default th:last-child { diff --git a/frontend/src/models/misc.ts b/frontend/src/models/misc.ts index 5478722c5..46af487c7 100644 --- a/frontend/src/models/misc.ts +++ b/frontend/src/models/misc.ts @@ -18,6 +18,7 @@ export type GetDocsResponse = { docs: Doc[]; totalDocuments: number; totalPages: number; + nextCursor: string; }; export type PromptProps = { @@ -28,7 +29,6 @@ export type PromptProps = { }; export type DocumentsProps = { - documents: Doc[] | null; handleDeleteDocument: (index: number, document: Doc) => void; }; diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts index e77c9e3b2..e3d22c02a 100644 --- a/frontend/src/preferences/preferenceApi.ts +++ b/frontend/src/preferences/preferenceApi.ts @@ -4,49 +4,53 @@ import { Doc, GetDocsResponse } from '../models/misc'; //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later. export async function getDocs( - sort?: string, - order?: string, - pageNumber?: number, - rowsPerPage?: number, - withPagination?: true, -): Promise; + sort = 'date', + order = 'desc', +): Promise { + try { + const response = await userService.getDocs(sort, order); + const data = await response.json(); -export async function getDocs( + const docs: Doc[] = []; + console.log(data); + data.forEach((doc: object) => { + docs.push(doc as Doc); + }); + + return docs; + } catch (error) { + console.log(error); + return null; + } +} + +export async function getDocsWithPagination( sort = 'date', order = 'desc', pageNumber = 1, - rowsPerPage = 5, - withPagination = false, -): Promise { + rowsPerPage = 10, +): Promise { try { - const response = await userService.getDocs( - sort, - order, - pageNumber, - rowsPerPage, - ); + const query = `sort=${sort}&order=${order}&page=${pageNumber}&rows=${rowsPerPage}`; + const response = await userService.getDocsWithPagination(query); const data = await response.json(); - console.log(data); - if (withPagination) { - const docs: Doc[] = []; - Array.isArray(data.paginated_docs) && - data.paginated_docs.forEach((doc: object) => { - docs.push(doc as Doc); - }); - - const totalDocuments = data.totalDocuments || 0; - const totalPages = data.totalPages || 0; - console.log(`totalDocuments: ${totalDocuments}`); - console.log(`totalPages: ${totalPages}`); - return { docs, totalDocuments, totalPages }; - } else { - const docs: Doc[] = []; - Array.isArray(data.documents) && - data.documents.forEach((doc: object) => { - docs.push(doc as Doc); - }); - return docs; - } + + const docs: Doc[] = []; + console.log(`data: ${data}`); + Array.isArray(data.paginated) && + data.paginated.forEach((doc: Doc) => { + docs.push(doc as Doc); + }); + console.log(`total: ${data.total}`); + console.log(`totalPages: ${data.totalPages}`); + console.log(`cursor: ${data.nextCursor}`); + console.log(`currentPage: ${data.currentPage}`); + return { + docs: docs, + totalDocuments: data.total, + totalPages: data.totalPages, + nextCursor: data.nextCursor, + }; } catch (error) { console.log(error); return null; diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx index fc21a8f13..8a35ca7d8 100644 --- a/frontend/src/settings/Documents.tsx +++ b/frontend/src/settings/Documents.tsx @@ -10,7 +10,7 @@ import caretSort from '../assets/caret-sort.svg'; import DropdownMenu from '../components/DropdownMenu'; import { Doc, DocumentsProps, ActiveState } from '../models/misc'; // Ensure ActiveState type is imported import SkeletonLoader from '../components/SkeletonLoader'; -import { getDocs } from '../preferences/preferenceApi'; +import { getDocs, getDocsWithPagination } from '../preferences/preferenceApi'; import { setSourceDocs } from '../preferences/preferenceSlice'; import Input from '../components/Input'; import Upload from '../upload/Upload'; // Import the Upload component @@ -33,13 +33,9 @@ const formatTokens = (tokens: number): string => { } }; -const Documents: React.FC = ({ - documents, - handleDeleteDocument, -}) => { +const Documents: React.FC = ({ handleDeleteDocument }) => { const { t } = useTranslation(); const dispatch = useDispatch(); - // State for search input const [searchTerm, setSearchTerm] = useState(''); // State for modal: active/inactive @@ -60,7 +56,6 @@ const Documents: React.FC = ({ ); // State for documents const currentDocuments = filteredDocuments ?? []; - const syncOptions = [ { label: 'Never', value: 'never' }, { label: 'Daily', value: 'daily' }, @@ -73,9 +68,9 @@ const Documents: React.FC = ({ pageNumber?: number, rows?: number, ) => { - console.log(`field: ${field}, pageNumber: ${pageNumber}, rows: ${rows}`); const page = pageNumber ?? currentPage; const rowsPerPg = rows ?? rowsPerPage; + if (field !== undefined) { if (field === sortField) { // Toggle sort order @@ -86,9 +81,9 @@ const Documents: React.FC = ({ setSortOrder('desc'); } } - getDocs(sortField, sortOrder, page, rowsPerPg, true) + getDocsWithPagination(sortField, sortOrder, page, rowsPerPg) .then((data) => { - console.log(data); + console.log('Data received from getDocsWithPagination:', data); dispatch(setSourceDocs(data ? data.docs : [])); setFetchedDocuments(data ? data.docs : []); setTotalPages(data ? data.totalPages : 0); @@ -99,6 +94,7 @@ const Documents: React.FC = ({ setLoading(false); }); }; + const handleManageSync = (doc: Doc, sync_frequency: string) => { setLoading(true); userService @@ -114,9 +110,13 @@ const Documents: React.FC = ({ setLoading(false); }); }; + useEffect(() => { - refreshDocs(sortField, currentPage, rowsPerPage); - }, []); + if (modalState === 'INACTIVE') { + refreshDocs(sortField, currentPage, rowsPerPage); + } + }, [modalState, sortField, currentPage, rowsPerPage]); + return (
@@ -190,7 +190,7 @@ const Documents: React.FC = ({ )} {Array.isArray(currentDocuments) && currentDocuments.map((document, index) => ( - + {document.name} {document.date} @@ -248,18 +248,24 @@ const Documents: React.FC = ({
)}
+ {/* Pagination component with props: + # Note: Every time the page changes, + the refreshDocs function is called with the updated page number and rows per page. + and reset cursor paginated query parameter to undefined. + */} { setCurrentPage(page); - refreshDocs(undefined, page, rowsPerPage); + refreshDocs(sortField, page, rowsPerPage); // Pass `true` to reset lastID if not using cursor }} onRowsPerPageChange={(rows) => { + console.log('Pagination - Rows per Page Change:', rows); setRowsPerPage(rows); - setCurrentPage(1); - refreshDocs(undefined, 1, rows); + setCurrentPage(1); // Reset to page 1 on rows per page change + refreshDocs(sortField, 1, rows); // Reset lastID for fresh pagination }} />
@@ -267,7 +273,7 @@ const Documents: React.FC = ({ }; Documents.propTypes = { - documents: PropTypes.array.isRequired, + //documents: PropTypes.array.isRequired, handleDeleteDocument: PropTypes.func.isRequired, }; diff --git a/frontend/src/settings/index.tsx b/frontend/src/settings/index.tsx index ea3d44281..24ea26812 100644 --- a/frontend/src/settings/index.tsx +++ b/frontend/src/settings/index.tsx @@ -70,12 +70,7 @@ export default function Settings() { case t('settings.general.label'): return ; case t('settings.documents.label'): - return ( - - ); + return ; case 'Widgets': return ( d.type?.toLowerCase() === 'local'), + Array.isArray(data) && + data?.find( + (d: Doc) => d.type?.toLowerCase() === 'local', + ), ), ); }); @@ -182,15 +185,21 @@ function Upload({ getDocs().then((data) => { dispatch(setSourceDocs(data)); const docIds = new Set( - sourceDocs?.map((doc: Doc) => (doc.id ? doc.id : null)), + (Array.isArray(sourceDocs) && + sourceDocs?.map((doc: Doc) => + doc.id ? doc.id : null, + )) || + [], ); - data?.map((updatedDoc: Doc) => { - if (updatedDoc.id && !docIds.has(updatedDoc.id)) { - //select the doc not present in the intersection of current Docs and fetched data - dispatch(setSelectedDocs(updatedDoc)); - return; - } - }); + if (data && Array.isArray(data.docs)) { + data.docs.map((updatedDoc: Doc) => { + if (updatedDoc.id && !docIds.has(updatedDoc.id)) { + // Select the doc not present in the intersection of current Docs and fetched data + dispatch(setSelectedDocs(updatedDoc)); + return; + } + }); + } }); setProgress( (progress) => From d59ffaf0bd6a3612331fc0d0cf960681851897c5 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Sat, 9 Nov 2024 14:57:15 -0500 Subject: [PATCH 17/26] fix sorting on /sources --- application/api/user/routes.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 7d44ce9bf..b3062005c 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -485,8 +485,6 @@ class CombinedJson(Resource): @api.doc(description="Provide JSON file with combined available indexes") def get(self): user = "local" - sort_field = request.args.get("sort", "date") # Default to 'date' - sort_order = request.args.get("order", "desc") # Default to 'desc' data = [ { "name": "default", @@ -500,7 +498,7 @@ def get(self): try: for index in sources_collection.find({"user": user}).sort( - sort_field, 1 if sort_order == "asc" else -1 + "date", -1 ): data.append( { From f3a005a66753149cc80e5adc95bdc7cfe0975b6a Mon Sep 17 00:00:00 2001 From: fadingNA Date: Sat, 9 Nov 2024 14:57:30 -0500 Subject: [PATCH 18/26] remove params on getDocs --- frontend/src/api/services/userService.ts | 3 +-- frontend/src/preferences/preferenceApi.ts | 7 ++----- frontend/src/settings/Documents.tsx | 3 +-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts index 82f322449..942318ae4 100644 --- a/frontend/src/api/services/userService.ts +++ b/frontend/src/api/services/userService.ts @@ -2,8 +2,7 @@ import apiClient from '../client'; import endpoints from '../endpoints'; const userService = { - getDocs: (sort: string, order: string): Promise => - apiClient.get(`${endpoints.USER.DOCS}?sort=${sort}&order=${order}`), + getDocs: (): Promise => apiClient.get(`${endpoints.USER.DOCS}`), getDocsWithPagination: (query: string): Promise => apiClient.get(`${endpoints.USER.DOCS_PAGINATED}?${query}`), checkDocs: (data: any): Promise => diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts index e3d22c02a..c1ac97de7 100644 --- a/frontend/src/preferences/preferenceApi.ts +++ b/frontend/src/preferences/preferenceApi.ts @@ -3,12 +3,9 @@ import userService from '../api/services/userService'; import { Doc, GetDocsResponse } from '../models/misc'; //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later. -export async function getDocs( - sort = 'date', - order = 'desc', -): Promise { +export async function getDocs(): Promise { try { - const response = await userService.getDocs(sort, order); + const response = await userService.getDocs(); const data = await response.json(); const docs: Doc[] = []; diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx index 8a35ca7d8..70cc86bab 100644 --- a/frontend/src/settings/Documents.tsx +++ b/frontend/src/settings/Documents.tsx @@ -83,8 +83,7 @@ const Documents: React.FC = ({ handleDeleteDocument }) => { } getDocsWithPagination(sortField, sortOrder, page, rowsPerPg) .then((data) => { - console.log('Data received from getDocsWithPagination:', data); - dispatch(setSourceDocs(data ? data.docs : [])); + //dispatch(setSourceDocs(data ? data.docs : [])); setFetchedDocuments(data ? data.docs : []); setTotalPages(data ? data.totalPages : 0); setTotalDocuments(data ? data.totalDocuments : 0); From 0837295bd3fa6d7fdc5b225c8b4c1d25e8271991 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Sat, 9 Nov 2024 16:24:21 -0500 Subject: [PATCH 19/26] fix table responsive issue --- frontend/src/index.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/index.css b/frontend/src/index.css index e8e9c9bdc..1c72035e8 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -50,11 +50,11 @@ body.dark { @layer components { .table-default { - @apply block w-full mx-auto table-fixed content-start justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray; + @apply block w-full mx-auto table-fixed content-start justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray overflow-auto; } .table-default th { - @apply p-4 w-1/4 font-normal text-gray-400 text-nowrap; /* Remove border-r */ + @apply p-4 w-1/3 font-normal text-gray-400 text-nowrap; /* Remove border-r */ } .table-default th:last-child { From e00c6f2c140962f0007e2a16c6b2fd4408192d68 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Sat, 9 Nov 2024 16:59:00 -0500 Subject: [PATCH 20/26] add min width for delete button, dropdown dakrtheme --- frontend/src/components/DocumentPagination.tsx | 2 +- frontend/src/index.css | 2 +- frontend/src/settings/Documents.tsx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/DocumentPagination.tsx b/frontend/src/components/DocumentPagination.tsx index b5d82456b..3edeff9df 100644 --- a/frontend/src/components/DocumentPagination.tsx +++ b/frontend/src/components/DocumentPagination.tsx @@ -48,7 +48,7 @@ const Pagination: React.FC = ({ onRowsPerPageChange(Number(e.target.value))} - className="border border-gray-300 rounded px-2 py-1 dark:bg-transparent dark:text-gray-50" + className="border border-gray-300 rounded px-2 py-1 dark:bg-dark-charcoal dark:text-gray-50" > {rowsPerPageOptions.map((option) => ( - ))} diff --git a/frontend/src/models/misc.ts b/frontend/src/models/misc.ts index 46af487c7..0d9c89314 100644 --- a/frontend/src/models/misc.ts +++ b/frontend/src/models/misc.ts @@ -29,6 +29,7 @@ export type PromptProps = { }; export type DocumentsProps = { + paginatedDocuments: Doc[] | null; handleDeleteDocument: (index: number, document: Doc) => void; }; diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx index 26af09d5d..eaee879e5 100644 --- a/frontend/src/settings/Documents.tsx +++ b/frontend/src/settings/Documents.tsx @@ -1,20 +1,20 @@ import React, { useState, useEffect } from 'react'; import PropTypes from 'prop-types'; -import { useTranslation } from 'react-i18next'; -import { useDispatch } from 'react-redux'; - import userService from '../api/services/userService'; import SyncIcon from '../assets/sync.svg'; import Trash from '../assets/trash.svg'; import caretSort from '../assets/caret-sort.svg'; import DropdownMenu from '../components/DropdownMenu'; -import { Doc, DocumentsProps, ActiveState } from '../models/misc'; // Ensure ActiveState type is imported import SkeletonLoader from '../components/SkeletonLoader'; -import { getDocs, getDocsWithPagination } from '../preferences/preferenceApi'; -import { setSourceDocs } from '../preferences/preferenceSlice'; import Input from '../components/Input'; import Upload from '../upload/Upload'; // Import the Upload component import Pagination from '../components/DocumentPagination'; +import { useTranslation } from 'react-i18next'; +import { useDispatch } from 'react-redux'; +import { Doc, DocumentsProps, ActiveState } from '../models/misc'; // Ensure ActiveState type is imported +import { getDocs, getDocsWithPagination } from '../preferences/preferenceApi'; +import { setSourceDocs } from '../preferences/preferenceSlice'; +import { setPaginatedDocuments } from '../preferences/preferenceSlice'; // Utility function to format numbers const formatTokens = (tokens: number): string => { @@ -33,7 +33,10 @@ const formatTokens = (tokens: number): string => { } }; -const Documents: React.FC = ({ handleDeleteDocument }) => { +const Documents: React.FC = ({ + paginatedDocuments, + handleDeleteDocument, +}) => { const { t } = useTranslation(); const dispatch = useDispatch(); // State for search input @@ -48,10 +51,9 @@ const Documents: React.FC = ({ handleDeleteDocument }) => { const [currentPage, setCurrentPage] = useState(1); const [rowsPerPage, setRowsPerPage] = useState(10); const [totalPages, setTotalPages] = useState(1); - const [totalDocuments, setTotalDocuments] = useState(0); - const [fetchedDocuments, setFetchedDocuments] = useState([]); + // const [totalDocuments, setTotalDocuments] = useState(0); // Filter documents based on the search term - const filteredDocuments = fetchedDocuments?.filter((document) => + const filteredDocuments = paginatedDocuments?.filter((document) => document.name.toLowerCase().includes(searchTerm.toLowerCase()), ); // State for documents @@ -84,9 +86,9 @@ const Documents: React.FC = ({ handleDeleteDocument }) => { getDocsWithPagination(sortField, sortOrder, page, rowsPerPg) .then((data) => { //dispatch(setSourceDocs(data ? data.docs : [])); - setFetchedDocuments(data ? data.docs : []); + dispatch(setPaginatedDocuments(data ? data.docs : [])); setTotalPages(data ? data.totalPages : 0); - setTotalDocuments(data ? data.totalDocuments : 0); + //setTotalDocuments(data ? data.totalDocuments : 0); }) .catch((error) => console.error(error)) .finally(() => { @@ -258,13 +260,12 @@ const Documents: React.FC = ({ handleDeleteDocument }) => { rowsPerPage={rowsPerPage} onPageChange={(page) => { setCurrentPage(page); - refreshDocs(sortField, page, rowsPerPage); // Pass `true` to reset lastID if not using cursor + refreshDocs(sortField, page, rowsPerPage); }} onRowsPerPageChange={(rows) => { - console.log('Pagination - Rows per Page Change:', rows); setRowsPerPage(rows); - setCurrentPage(1); // Reset to page 1 on rows per page change - refreshDocs(sortField, 1, rows); // Reset lastID for fresh pagination + setCurrentPage(1); + refreshDocs(sortField, 1, rows); }} />
diff --git a/frontend/src/settings/index.tsx b/frontend/src/settings/index.tsx index 24ea26812..15c7ce086 100644 --- a/frontend/src/settings/index.tsx +++ b/frontend/src/settings/index.tsx @@ -8,6 +8,8 @@ import i18n from '../locale/i18n'; import { Doc } from '../models/misc'; import { selectSourceDocs, + selectPaginatedDocuments, + setPaginatedDocuments, setSourceDocs, } from '../preferences/preferenceSlice'; import Analytics from './Analytics'; @@ -26,20 +28,29 @@ export default function Settings() { ); const documents = useSelector(selectSourceDocs); + const paginatedDocuments = useSelector(selectPaginatedDocuments); const updateWidgetScreenshot = (screenshot: File | null) => { setWidgetScreenshot(screenshot); }; + const updateDocumentsList = (documents: Doc[], index: number) => [ + ...documents.slice(0, index), + ...documents.slice(index + 1), + ]; + const handleDeleteClick = (index: number, doc: Doc) => { userService .deletePath(doc.id ?? '') .then((response) => { if (response.ok && documents) { - const updatedDocuments = [ - ...documents.slice(0, index), - ...documents.slice(index + 1), - ]; - dispatch(setSourceDocs(updatedDocuments)); + if (paginatedDocuments) { + dispatch( + setPaginatedDocuments( + updateDocumentsList(paginatedDocuments, index), + ), + ); + } + dispatch(setSourceDocs(updateDocumentsList(documents, index))); } }) .catch((error) => console.error(error)); @@ -70,7 +81,12 @@ export default function Settings() { case t('settings.general.label'): return ; case t('settings.documents.label'): - return ; + return ( + + ); case 'Widgets': return ( Date: Sun, 10 Nov 2024 14:53:52 -0500 Subject: [PATCH 24/26] add paginated to store, and fix upload docs property --- frontend/src/store.ts | 1 + frontend/src/upload/Upload.tsx | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/src/store.ts b/frontend/src/store.ts index 5843d4932..8f426ed61 100644 --- a/frontend/src/store.ts +++ b/frontend/src/store.ts @@ -38,6 +38,7 @@ const preloadedState: { preference: Preference } = { }, ], modalState: 'INACTIVE', + paginatedDocuments: null, }, }; const store = configureStore({ diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 443127c05..4f31ea0ba 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -191,8 +191,8 @@ function Upload({ )) || [], ); - if (data && Array.isArray(data.docs)) { - data.docs.map((updatedDoc: Doc) => { + if (data && Array.isArray(data)) { + data.map((updatedDoc: Doc) => { if (updatedDoc.id && !docIds.has(updatedDoc.id)) { // Select the doc not present in the intersection of current Docs and fetched data dispatch(setSelectedDocs(updatedDoc)); From 32c67c2a0213a829a02f366084c41024f7e45cd9 Mon Sep 17 00:00:00 2001 From: fadingNA Date: Sun, 10 Nov 2024 15:33:19 -0500 Subject: [PATCH 25/26] add property sync and etc ... to align with original source --- application/api/user/routes.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 46ed15cde..37a3f7a15 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -465,10 +465,17 @@ def get(self): paginated_docs = [] for doc in documents: - print(doc) - doc["id"] = str(doc["_id"]) - del doc["_id"] - paginated_docs.append(doc) + doc_data = { + "id": str(doc["_id"]), + "name": doc.get("name", ""), + "date": doc.get("date", ""), + "model": settings.EMBEDDINGS_NAME, + "location": "local", + "tokens": doc.get("tokens", ""), + "retriever": doc.get("retriever", "classic"), + "syncFrequency": doc.get("sync_frequency", ""), + } + paginated_docs.append(doc_data) response = { "total": total_documents, @@ -499,9 +506,7 @@ def get(self): ] try: - for index in sources_collection.find({"user": user}).sort( - "date", -1 - ): + for index in sources_collection.find({"user": user}).sort("date", -1): data.append( { "id": str(index["_id"]), From 6974db5fd89711331a4565ae7a7cc4ee7619f06a Mon Sep 17 00:00:00 2001 From: fadingNA Date: Sun, 10 Nov 2024 15:33:33 -0500 Subject: [PATCH 26/26] fix manage sync --- frontend/src/index.css | 2 +- frontend/src/settings/Documents.tsx | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/frontend/src/index.css b/frontend/src/index.css index 6de42c244..7c9cc02ce 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -54,7 +54,7 @@ body.dark { } .table-default th { - @apply p-4 w-1/3 font-normal text-gray-400 text-nowrap; /* Remove border-r */ + @apply p-4 w-full font-normal text-gray-400 text-nowrap; /* Remove border-r */ } .table-default th:last-child { diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx index eaee879e5..f91a33559 100644 --- a/frontend/src/settings/Documents.tsx +++ b/frontend/src/settings/Documents.tsx @@ -58,6 +58,7 @@ const Documents: React.FC = ({ ); // State for documents const currentDocuments = filteredDocuments ?? []; + console.log('currentDocuments', currentDocuments); const syncOptions = [ { label: 'Never', value: 'never' }, { label: 'Daily', value: 'daily' }, @@ -101,12 +102,25 @@ const Documents: React.FC = ({ userService .manageSync({ source_id: doc.id, sync_frequency }) .then(() => { + // First, fetch the updated source docs return getDocs(); }) .then((data) => { dispatch(setSourceDocs(data)); + return getDocsWithPagination( + sortField, + sortOrder, + currentPage, + rowsPerPage, + ); }) - .catch((error) => console.error(error)) + .then((paginatedData) => { + dispatch( + setPaginatedDocuments(paginatedData ? paginatedData.docs : []), + ); + setTotalPages(paginatedData ? paginatedData.totalPages : 0); + }) + .catch((error) => console.error('Error in handleManageSync:', error)) .finally(() => { setLoading(false); });