diff --git a/commanddash/lib/agent/agent_handler.dart b/commanddash/lib/agent/agent_handler.dart index 2d2aa7c..30a31ee 100644 --- a/commanddash/lib/agent/agent_handler.dart +++ b/commanddash/lib/agent/agent_handler.dart @@ -14,7 +14,7 @@ class AgentHandler { final Map outputs; final List> steps; final String githubAccessToken; - final String geminiApiKey; + final String? geminiApiKey; final String agentName; final String agentVersion; final List dataSources; @@ -59,7 +59,9 @@ class AgentHandler { outputs: outputs, steps: steps, githubAccessToken: json['auth_details']['github_token'], - geminiApiKey: json['auth_details']['key'], + geminiApiKey: json['auth_details']['key'].isEmpty + ? null + : json['auth_details']['key'], agentName: json['agent_name'], agentVersion: json['agent_version'], isTest: json['testing'] ?? false, @@ -116,6 +118,7 @@ class AgentHandler { taskAssist, geminiRepository, dashRepository); final ChatStep chatStep = ChatStep( + agentName: agentName, outputIds: [], inputs: inputs, outputs: {}, diff --git a/commanddash/lib/agent/step_model.dart b/commanddash/lib/agent/step_model.dart index 296602a..17393dc 100644 --- a/commanddash/lib/agent/step_model.dart +++ b/commanddash/lib/agent/step_model.dart @@ -45,8 +45,8 @@ abstract class Step { }).toList(); return PromptQueryStep.fromJson( json, + agentName, (json['prompt'] as String), - // .replacePlaceholder(inputs, outputs), outputsList, inputs, outputs, diff --git a/commanddash/lib/repositories/client/dio_client.dart b/commanddash/lib/repositories/client/dio_client.dart index 705e806..1e2c70e 100644 --- a/commanddash/lib/repositories/client/dio_client.dart +++ b/commanddash/lib/repositories/client/dio_client.dart @@ -1,12 +1,17 @@ import 'dart:async'; +import 'package:commanddash/runner.dart'; import 'package:dio/dio.dart'; typedef RefreshAccessToken = Future> Function(); -Dio getClient(String accessToken, RefreshAccessToken updateAccessToken) { +Dio getClient( + String accessToken, + RefreshAccessToken updateAccessToken, +) { final dio = Dio(); dio.options.baseUrl = 'https://api.commanddash.dev'; + dio.options.headers['Engine-Version'] = version; dio.interceptors.add(CustomInterceptor( dio: dio, accessToken: accessToken, diff --git a/commanddash/lib/repositories/gemini_repository.dart b/commanddash/lib/repositories/gemini_repository.dart index 40dcdbc..06c1269 100644 --- a/commanddash/lib/repositories/gemini_repository.dart +++ b/commanddash/lib/repositories/gemini_repository.dart @@ -18,7 +18,7 @@ class UnknownException implements Exception { } class GeminiRepository { - final String apiKey; + final String? apiKey; late Dio dio; final int characterLimit = 120000.tokenized; @@ -26,19 +26,22 @@ class GeminiRepository { GeminiRepository(this.apiKey, this.dio); factory GeminiRepository.fromKeys( - String keys, String githubAccessToken, TaskAssist taskAssist) { + String? apiKey, String githubAccessToken, TaskAssist taskAssist) { final client = getClient( githubAccessToken, () async => taskAssist .processOperation(kind: 'refresh_access_token', args: {})); - return GeminiRepository(keys, client); + return GeminiRepository(apiKey, client); } Future> getCodeEmbeddings( String value, ) async { + if (apiKey == null) { + throw Exception('This functionality is not supported without Gemini Key'); + } try { - final model = GenerativeModel(model: 'embedding-001', apiKey: apiKey); + final model = GenerativeModel(model: 'embedding-001', apiKey: apiKey!); final content = Content.text(value); final result = await model.embedContent( @@ -62,8 +65,11 @@ class GeminiRepository { Future>> getCodeBatchEmbeddings( List> code) async { + if (apiKey == null) { + throw Exception('This functionality is not supported without Gemini Key'); + } try { - final model = GenerativeModel(model: 'embedding-001', apiKey: apiKey); + final model = GenerativeModel(model: 'embedding-001', apiKey: apiKey!); final embedRequest = code .map((value) => EmbedContentRequest(Content.text(value['content']), title: value['title'], taskType: TaskType.retrievalDocument)) @@ -85,8 +91,11 @@ class GeminiRepository { } Future> getStringEmbeddings(String value) async { + if (apiKey == null) { + throw Exception('This functionality is not supported without Gemini Key'); + } try { - final model = GenerativeModel(model: 'embedding-001', apiKey: apiKey); + final model = GenerativeModel(model: 'embedding-001', apiKey: apiKey!); final content = Content.text(value); final result = await model.embedContent( content, @@ -108,9 +117,11 @@ class GeminiRepository { Future>> getStringBatchEmbeddings( List values) async { - //TODO: update to batch embed + if (apiKey == null) { + throw Exception('This functionality is not supported without Gemini Key'); + } try { - final response = await HttpApiClient(apiKey: apiKey).makeRequest( + final response = await HttpApiClient(apiKey: apiKey!).makeRequest( Uri.https('generativelanguage.googleapis.com').resolveUri(Uri( pathSegments: [ 'v1', @@ -150,61 +161,66 @@ class GeminiRepository { } } - Future getCompletion( - String messages, - ) async { - late final GenerateContentResponse? response; - try { - response = - await _getGeminiFlashCompletionResponse('gemini-1.5-flash', messages); - } on ServerException catch (e) { - if (e.message.contains( - 'found for API version v1beta, or is not supported for GenerateContent')) { - response = - await _getGeminiFlashCompletionResponse('gemini-pro', messages); + Future getCompletion(String message, String agentName, + {String? command}) async { + String response; + if (apiKey != null) { + try { + response = await _getGeminiFlashChatCompletionResponse( + 'gemini-1.5-flash', [], message, apiKey!); + } on ServerException catch (e) { + // throw GenerativeAIException('recitation'); + if (e.message.contains('recitation') || + e.message + .contains('User location is not supported for the API use')) { + response = await getApiCompletionResponse([], message, + agentName: agentName, command: command); + } else { + rethrow; + } } - } - if (response != null && response.text != null) { - return response.text!; } else { - throw ModelException("No response recieved from gemini"); + response = await getApiCompletionResponse([], message, + agentName: agentName, command: command); } - } - Future _getGeminiFlashCompletionResponse( - String modelCode, String messages) async { - final model = GenerativeModel(model: modelCode, apiKey: apiKey); - final content = [Content.text(messages)]; - return model.generateContent(content); + return response; } Future getChatCompletion( List messages, String lastMessage, { + required String agent, + String? command, String? systemPrompt, }) async { String response; - - try { - // throw GenerativeAIException('recitation'); - response = await _getGeminiFlashChatCompletionResponse( - 'gemini-1.5-flash', messages, lastMessage, - systemPrompt: systemPrompt); - } on GenerativeAIException catch (e) { - if (e.message.contains('recitation') || - e.message - .contains('User location is not supported for the API use')) { - response = await getApiCompletionResponse(messages, lastMessage, + if (apiKey != null) { + try { + response = await _getGeminiFlashChatCompletionResponse( + 'gemini-1.5-flash', messages, lastMessage, apiKey!, systemPrompt: systemPrompt); - } else { - rethrow; + } on GenerativeAIException catch (e) { + // throw GenerativeAIException('recitation'); + if (e.message.contains('recitation') || + e.message + .contains('User location is not supported for the API use')) { + response = await getApiCompletionResponse(messages, lastMessage, + systemPrompt: systemPrompt, agentName: agent, command: command); + } else { + rethrow; + } } + } else { + response = await getApiCompletionResponse(messages, lastMessage, + systemPrompt: systemPrompt, agentName: agent, command: command); } + return response; } - Future _getGeminiFlashChatCompletionResponse( - String modelCode, List messages, String lastMessage, + Future _getGeminiFlashChatCompletionResponse(String modelCode, + List messages, String lastMessage, String apiKey, {String? systemPrompt}) async { // system intructions are not being adapted that well by Gemini models. // final Content? systemInstruction = @@ -234,7 +250,9 @@ class GeminiRepository { Future getApiCompletionResponse( List messages, String lastMessage, - {String? systemPrompt}) async { + {required String agentName, + String? command, + String? systemPrompt}) async { final List> message = []; if (systemPrompt != null) { @@ -251,14 +269,11 @@ class GeminiRepository { final response = await dio.post( '/ai/agent/answer', - data: { - 'message': message, - }, + data: {'message': message, 'agent': agentName, 'command': command}, ); if (response.statusCode == 200) { return response.data['content'] as String; } - throw ModelException( - '${response.statusCode}: ${jsonDecode(response.data)['message']}'); + throw ModelException('${response.statusCode}: ${response.data['message']}'); } } diff --git a/commanddash/lib/runner.dart b/commanddash/lib/runner.dart index 3551030..aa08878 100644 --- a/commanddash/lib/runner.dart +++ b/commanddash/lib/runner.dart @@ -21,6 +21,8 @@ class ProcessCommand extends Command { } } +final version = '0.1.5'; + class VersionCommand extends Command { @override final String name = "version"; @@ -33,7 +35,7 @@ class VersionCommand extends Command { @override void run() { /// It is not possible to fetch version from pubspec.yaml hence assigning manually - print('0.1.4'); + print(version); } } diff --git a/commanddash/lib/steps/chat/chat_step.dart b/commanddash/lib/steps/chat/chat_step.dart index 0b8fadc..0081fb3 100644 --- a/commanddash/lib/steps/chat/chat_step.dart +++ b/commanddash/lib/steps/chat/chat_step.dart @@ -14,6 +14,7 @@ import 'package:commanddash/steps/steps_utils.dart'; import 'package:commanddash/utils/embedding_utils.dart'; class ChatStep extends Step { + final String agentName; final List existingMessages; final String lastMessage; final Map inputs; @@ -23,7 +24,8 @@ class ChatStep extends Step { final String? existingDocuments; //TODO: See if we can save data source result output only ChatStep( - {required List outputIds, + {required this.agentName, + required List outputIds, required this.existingMessages, required this.inputs, this.systemPrompt, @@ -180,6 +182,7 @@ class ChatStep extends Step { ...existingMessages ], prompt, + agent: agentName, systemPrompt: systemPrompt, ); return [DefaultOutput(response)]; diff --git a/commanddash/lib/steps/prompt_query/prompt_query_step.dart b/commanddash/lib/steps/prompt_query/prompt_query_step.dart index a4adbfd..74c650c 100644 --- a/commanddash/lib/steps/prompt_query/prompt_query_step.dart +++ b/commanddash/lib/steps/prompt_query/prompt_query_step.dart @@ -10,12 +10,14 @@ import 'package:commanddash/steps/prompt_query/prompt_response_parsers.dart'; import 'package:commanddash/steps/steps_utils.dart'; class PromptQueryStep extends Step { + final String agentName; final String query; final List outputs; Map inputs; Map outputsUntilNow; //TODO:rename needed PromptQueryStep( - {required List? outputIds, + {required this.agentName, + required List? outputIds, required this.outputs, required this.query, required this.inputs, @@ -25,12 +27,14 @@ class PromptQueryStep extends Step { factory PromptQueryStep.fromJson( Map json, + String agentName, String query, List outputs, Map inputs, Map outputsUntilNow, ) { return PromptQueryStep( + agentName: agentName, outputIds: (json['outputs'] as List).map((e) => e.toString()).toList(), outputs: outputs, @@ -187,7 +191,8 @@ class PromptQueryStep extends Step { args: ProcessingFilesLoader(filesInvolved, message: 'Preparing Result') .toJson(), timeoutKind: TimeoutKind.sync); - final response = await generationRepository.getCompletion(prompt); + final response = await generationRepository.getCompletion(prompt, agentName, + command: 'untracked'); final result = []; for (Output output in outputs) { if (output is CodeOutput) { diff --git a/commanddash/pubspec.yaml b/commanddash/pubspec.yaml index 8d41255..f5e59f4 100644 --- a/commanddash/pubspec.yaml +++ b/commanddash/pubspec.yaml @@ -1,6 +1,6 @@ name: commanddash description: Processing engine for CommandDash clients like the IDEs -version: 0.1.4 +version: 0.1.5 repository: https://github.com/Welltested-AI/commanddash environment: diff --git a/dash_agent/CHANGELOG.md b/dash_agent/CHANGELOG.md index b2fcd17..1b7974f 100644 --- a/dash_agent/CHANGELOG.md +++ b/dash_agent/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.4.0 + +* **Github Data Object Update**: Updated github object to allow extraction of only code, issues, or both. When defining `Github` object via `WebDataObject.fromGithub`, you can pass list of `GithubExtract` enums containing all the intended data types that you want to be extracted and index. For example, passing `[GithubExtract.code, GithubExtract.issues]` will index both code and issues. +* **Deep Crawling Web Page Support**: Added support for deep crawling the website to extract sites data without the need of providing explicit sitemap. You can use `WebDataObject.fromWebPage` along with base url of the website that you want to index. Additionally, you will also need to pass `deepCrawl` to `true` to enable deep crawling. By default the value is `false`, which means it will only index the shared url page only + ## 0.3.0 * **Github Data Object Support**: Added support to extract Github repo data conviniently. You can use `WebDataObject.fromGithub` for indexing a specific repo code and issues now. diff --git a/dash_agent/README.md b/dash_agent/README.md index bfeec62..cc1c9f0 100644 --- a/dash_agent/README.md +++ b/dash_agent/README.md @@ -93,9 +93,10 @@ class DocsDataSource extends DataSource { /// URLs or sitemaps or even github repos in the object. @override List get webObjects => - [WebDataObject.fromWebPage('https://sampleurl.com'), + [WebDataObject.fromWebPage('https://sampleurl.com'), + WebDataObject.fromWebPage('https://sampleurl.com', deepCrawl: true), // to index all the pages of sampleurl.com WebDataObject.fromSiteMap('https://sampleurl.com/sitemap.xml'), - WebDataObject.fromGithub('https://github.com/user/repo', '', CodeFilter(pathRegex: '.*\.dart'), IssueFilter(labels: ['bug']))]; + WebDataObject.fromGithub('https://github.com/user/repo', '', codeFilter: CodeFilter(pathRegex: '.*\.dart'), issueFilter: IssueFilter(labels: ['bug']))]; } ``` diff --git a/dash_agent/lib/data/objects/web_data_object.dart b/dash_agent/lib/data/objects/web_data_object.dart index 0b8e2dc..048e547 100644 --- a/dash_agent/lib/data/objects/web_data_object.dart +++ b/dash_agent/lib/data/objects/web_data_object.dart @@ -10,15 +10,17 @@ abstract class WebDataObject { const WebDataObject(); /// static method to create and return [WebPage]. It takes `url` of web page as - /// an argument + /// an argument. Additionally, you can provide `deepCrawl` argument. If + /// `deepCrawl` set to true. It will crawl the whole website of the shared + /// url and index all of the pages in it. /// /// Example: /// ```dart /// final webUrl = 'https://sampleurl.com'; - /// final webPageObject = WebDataObject.fromWebPage(webUrl); + /// final webPageObject = WebDataObject.fromWebPage(webUrl, deepCrawl = false); /// ``` - static WebPage fromWebPage(String url) { - return WebPage(url); + static WebPage fromWebPage(String url, {bool deepCrawl = false}) { + return WebPage(url, deepCrawl: deepCrawl); } /// static method to create and return [SiteMap]. It takes `xml` as an argument @@ -82,22 +84,29 @@ abstract class WebDataObject { /// Example: /// ```dart /// final webUrl = 'https://sampleurl.com'; -/// final webPageObject = WebDataObject.fromWebPage(webUrl); +/// final webPageObject = WebDataObject.fromWebPage(webUrl, deepCrawl = false); /// ``` class WebPage extends WebDataObject { /// Creates a new [WebPage] object. /// - /// * url: URL of the web page that contains the content that you want to save for referencing. - WebPage(this.url); + /// * url: URL of the web page that contains the content that you want to save + /// for referencing. + /// * deepCrawl: Boolean value for enabling deep crawl of the website + /// if set to `true`. Default is `false` + WebPage(this.url, {this.deepCrawl = false}); /// url of the web page that contains the content that you want to save for referencing. final String url; + /// Boolean value for enabling deep crawl of the website if set to `true`. Default is `false` + final bool deepCrawl; + @override Future> process() async { + final objectType = deepCrawl ? 'deep_crawl_page' : 'web_page'; return { 'id': hashCode.toString(), - 'type': 'web_page', + 'type': objectType, 'url': url, 'version': minCliVersion }; diff --git a/dash_agent/pubspec.yaml b/dash_agent/pubspec.yaml index 3d3e163..d67d6e2 100644 --- a/dash_agent/pubspec.yaml +++ b/dash_agent/pubspec.yaml @@ -1,6 +1,6 @@ name: dash_agent description: "Framework package to create and publish dash agents for commanddash" -version: 0.3.0 +version: 0.4.0 homepage: https://www.commanddash.io repository: https://github.com/CommandDash/packages/tree/main/dash_agent diff --git a/dash_cli/pubspec.lock b/dash_cli/pubspec.lock index d5efd8e..4a4d88f 100644 --- a/dash_cli/pubspec.lock +++ b/dash_cli/pubspec.lock @@ -33,6 +33,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.1.0" + archive: + dependency: transitive + description: + name: archive + sha256: cb6a278ef2dbb298455e1a713bda08524a175630ec643a242c399c932a0a1f7d + url: "https://pub.dev" + source: hosted + version: "3.6.1" args: dependency: "direct main" description: @@ -193,6 +201,14 @@ packages: url: "https://pub.dev" source: hosted version: "4.0.2" + image: + dependency: "direct main" + description: + name: image + sha256: "2237616a36c0d69aef7549ab439b833fb7f9fb9fc861af2cc9ac3eedddd69ca8" + url: "https://pub.dev" + source: hosted + version: "4.2.0" interact: dependency: "direct main" description: @@ -289,6 +305,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.9.0" + petitparser: + dependency: transitive + description: + name: petitparser + sha256: c15605cd28af66339f8eb6fbe0e541bfe2d1b72d5825efc6598f3e0a31b9ad27 + url: "https://pub.dev" + source: hosted + version: "6.0.2" pointycastle: dependency: transitive description: @@ -489,6 +513,14 @@ packages: url: "https://pub.dev" source: hosted version: "5.2.0" + xml: + dependency: transitive + description: + name: xml + sha256: b015a8ad1c488f66851d762d3090a21c600e479dc75e68328c52774040cf9226 + url: "https://pub.dev" + source: hosted + version: "6.5.0" yaml: dependency: "direct main" description: