Merge pull request #95 from CommandDash/rel-1.0.5

Rel 1.0.5

commanddash/lib/agent/agent_handler.dart

@@ -14,7 +14,7 @@ class AgentHandler {
   final Map<String, Output> outputs;
   final List<Map<String, dynamic>> steps;
   final String githubAccessToken;
-  final String geminiApiKey;
+  final String? geminiApiKey;
   final String agentName;
   final String agentVersion;
   final List<String> dataSources;
@@ -59,7 +59,9 @@ class AgentHandler {
         outputs: outputs,
         steps: steps,
         githubAccessToken: json['auth_details']['github_token'],
-        geminiApiKey: json['auth_details']['key'],
+        geminiApiKey: json['auth_details']['key'].isEmpty
+            ? null
+            : json['auth_details']['key'],
         agentName: json['agent_name'],
         agentVersion: json['agent_version'],
         isTest: json['testing'] ?? false,
@@ -116,6 +118,7 @@ class AgentHandler {
         taskAssist, geminiRepository, dashRepository);
 
     final ChatStep chatStep = ChatStep(
+        agentName: agentName,
         outputIds: [],
         inputs: inputs,
         outputs: {},

commanddash/lib/agent/step_model.dart

@@ -45,8 +45,8 @@ abstract class Step {
         }).toList();
         return PromptQueryStep.fromJson(
           json,
+          agentName,
           (json['prompt'] as String),
-          // .replacePlaceholder(inputs, outputs),
           outputsList,
           inputs,
           outputs,

commanddash/lib/repositories/client/dio_client.dart

@@ -1,12 +1,17 @@
 import 'dart:async';
 
+import 'package:commanddash/runner.dart';
 import 'package:dio/dio.dart';
 
 typedef RefreshAccessToken = Future<Map<String, dynamic>> Function();
 
-Dio getClient(String accessToken, RefreshAccessToken updateAccessToken) {
+Dio getClient(
+  String accessToken,
+  RefreshAccessToken updateAccessToken,
+) {
   final dio = Dio();
   dio.options.baseUrl = 'https://api.commanddash.dev';
+  dio.options.headers['Engine-Version'] = version;
   dio.interceptors.add(CustomInterceptor(
       dio: dio,
       accessToken: accessToken,

commanddash/lib/repositories/gemini_repository.dart

@@ -18,27 +18,30 @@ class UnknownException implements Exception {
 }
 
 class GeminiRepository {
-  final String apiKey;
+  final String? apiKey;
   late Dio dio;
 
   final int characterLimit = 120000.tokenized;
 
   GeminiRepository(this.apiKey, this.dio);
 
   factory GeminiRepository.fromKeys(
-      String keys, String githubAccessToken, TaskAssist taskAssist) {
+      String? apiKey, String githubAccessToken, TaskAssist taskAssist) {
     final client = getClient(
         githubAccessToken,
         () async => taskAssist
             .processOperation(kind: 'refresh_access_token', args: {}));
-    return GeminiRepository(keys, client);
+    return GeminiRepository(apiKey, client);
   }
 
   Future<List<double>> getCodeEmbeddings(
     String value,
   ) async {
+    if (apiKey == null) {
+      throw Exception('This functionality is not supported without Gemini Key');
+    }
     try {
-      final model = GenerativeModel(model: 'embedding-001', apiKey: apiKey);
+      final model = GenerativeModel(model: 'embedding-001', apiKey: apiKey!);
       final content = Content.text(value);
 
       final result = await model.embedContent(
@@ -62,8 +65,11 @@ class GeminiRepository {
 
   Future<List<List<double>>> getCodeBatchEmbeddings(
       List<Map<String, dynamic>> code) async {
+    if (apiKey == null) {
+      throw Exception('This functionality is not supported without Gemini Key');
+    }
     try {
-      final model = GenerativeModel(model: 'embedding-001', apiKey: apiKey);
+      final model = GenerativeModel(model: 'embedding-001', apiKey: apiKey!);
       final embedRequest = code
           .map((value) => EmbedContentRequest(Content.text(value['content']),
               title: value['title'], taskType: TaskType.retrievalDocument))
@@ -85,8 +91,11 @@ class GeminiRepository {
   }
 
   Future<List<double>> getStringEmbeddings(String value) async {
+    if (apiKey == null) {
+      throw Exception('This functionality is not supported without Gemini Key');
+    }
     try {
-      final model = GenerativeModel(model: 'embedding-001', apiKey: apiKey);
+      final model = GenerativeModel(model: 'embedding-001', apiKey: apiKey!);
       final content = Content.text(value);
       final result = await model.embedContent(
         content,
@@ -108,9 +117,11 @@ class GeminiRepository {
 
   Future<List<List<double>>> getStringBatchEmbeddings(
       List<String> values) async {
-    //TODO: update to batch embed
+    if (apiKey == null) {
+      throw Exception('This functionality is not supported without Gemini Key');
+    }
     try {
-      final response = await HttpApiClient(apiKey: apiKey).makeRequest(
+      final response = await HttpApiClient(apiKey: apiKey!).makeRequest(
           Uri.https('generativelanguage.googleapis.com').resolveUri(Uri(
               pathSegments: [
                 'v1',
@@ -150,61 +161,66 @@ class GeminiRepository {
     }
   }
 
-  Future<String> getCompletion(
-    String messages,
-  ) async {
-    late final GenerateContentResponse? response;
-    try {
-      response =
-          await _getGeminiFlashCompletionResponse('gemini-1.5-flash', messages);
-    } on ServerException catch (e) {
-      if (e.message.contains(
-          'found for API version v1beta, or is not supported for GenerateContent')) {
-        response =
-            await _getGeminiFlashCompletionResponse('gemini-pro', messages);
+  Future<String> getCompletion(String message, String agentName,
+      {String? command}) async {
+    String response;
+    if (apiKey != null) {
+      try {
+        response = await _getGeminiFlashChatCompletionResponse(
+            'gemini-1.5-flash', [], message, apiKey!);
+      } on ServerException catch (e) {
+        // throw GenerativeAIException('recitation');
+        if (e.message.contains('recitation') ||
+            e.message
+                .contains('User location is not supported for the API use')) {
+          response = await getApiCompletionResponse([], message,
+              agentName: agentName, command: command);
+        } else {
+          rethrow;
+        }
       }
-    }
-    if (response != null && response.text != null) {
-      return response.text!;
     } else {
-      throw ModelException("No response recieved from gemini");
+      response = await getApiCompletionResponse([], message,
+          agentName: agentName, command: command);
     }
-  }
 
-  Future<GenerateContentResponse> _getGeminiFlashCompletionResponse(
-      String modelCode, String messages) async {
-    final model = GenerativeModel(model: modelCode, apiKey: apiKey);
-    final content = [Content.text(messages)];
-    return model.generateContent(content);
+    return response;
   }
 
   Future<String> getChatCompletion(
     List<ChatMessage> messages,
     String lastMessage, {
+    required String agent,
+    String? command,
     String? systemPrompt,
   }) async {
     String response;
-
-    try {
-      // throw GenerativeAIException('recitation');
-      response = await _getGeminiFlashChatCompletionResponse(
-          'gemini-1.5-flash', messages, lastMessage,
-          systemPrompt: systemPrompt);
-    } on GenerativeAIException catch (e) {
-      if (e.message.contains('recitation') ||
-          e.message
-              .contains('User location is not supported for the API use')) {
-        response = await getApiCompletionResponse(messages, lastMessage,
+    if (apiKey != null) {
+      try {
+        response = await _getGeminiFlashChatCompletionResponse(
+            'gemini-1.5-flash', messages, lastMessage, apiKey!,
             systemPrompt: systemPrompt);
-      } else {
-        rethrow;
+      } on GenerativeAIException catch (e) {
+        // throw GenerativeAIException('recitation');
+        if (e.message.contains('recitation') ||
+            e.message
+                .contains('User location is not supported for the API use')) {
+          response = await getApiCompletionResponse(messages, lastMessage,
+              systemPrompt: systemPrompt, agentName: agent, command: command);
+        } else {
+          rethrow;
+        }
       }
+    } else {
+      response = await getApiCompletionResponse(messages, lastMessage,
+          systemPrompt: systemPrompt, agentName: agent, command: command);
     }
+
     return response;
   }
 
-  Future<String> _getGeminiFlashChatCompletionResponse(
-      String modelCode, List<ChatMessage> messages, String lastMessage,
+  Future<String> _getGeminiFlashChatCompletionResponse(String modelCode,
+      List<ChatMessage> messages, String lastMessage, String apiKey,
       {String? systemPrompt}) async {
     // system intructions are not being adapted that well by Gemini models.
     // final Content? systemInstruction =
@@ -234,7 +250,9 @@ class GeminiRepository {
 
   Future<String> getApiCompletionResponse(
       List<ChatMessage> messages, String lastMessage,
-      {String? systemPrompt}) async {
+      {required String agentName,
+      String? command,
+      String? systemPrompt}) async {
     final List<Map<String, String>> message = [];
 
     if (systemPrompt != null) {
@@ -251,14 +269,11 @@ class GeminiRepository {
 
     final response = await dio.post(
       '/ai/agent/answer',
-      data: {
-        'message': message,
-      },
+      data: {'message': message, 'agent': agentName, 'command': command},
     );
     if (response.statusCode == 200) {
       return response.data['content'] as String;
     }
-    throw ModelException(
-        '${response.statusCode}: ${jsonDecode(response.data)['message']}');
+    throw ModelException('${response.statusCode}: ${response.data['message']}');
   }
 }

commanddash/lib/runner.dart

@@ -21,6 +21,8 @@ class ProcessCommand extends Command {
   }
 }
 
+final version = '0.1.5';
+
 class VersionCommand extends Command {
   @override
   final String name = "version";
@@ -33,7 +35,7 @@ class VersionCommand extends Command {
   @override
   void run() {
     /// It is not possible to fetch version from pubspec.yaml hence assigning manually
-    print('0.1.4');
+    print(version);
   }
 }
 

commanddash/lib/steps/chat/chat_step.dart

@@ -14,6 +14,7 @@ import 'package:commanddash/steps/steps_utils.dart';
 import 'package:commanddash/utils/embedding_utils.dart';
 
 class ChatStep extends Step {
+  final String agentName;
   final List<ChatMessage> existingMessages;
   final String lastMessage;
   final Map<String, Input> inputs;
@@ -23,7 +24,8 @@ class ChatStep extends Step {
   final String?
       existingDocuments; //TODO: See if we can save data source result output only
   ChatStep(
-      {required List<String> outputIds,
+      {required this.agentName,
+      required List<String> outputIds,
       required this.existingMessages,
       required this.inputs,
       this.systemPrompt,
@@ -180,6 +182,7 @@ class ChatStep extends Step {
         ...existingMessages
       ],
       prompt,
+      agent: agentName,
       systemPrompt: systemPrompt,
     );
     return [DefaultOutput(response)];

commanddash/lib/steps/prompt_query/prompt_query_step.dart

@@ -10,12 +10,14 @@ import 'package:commanddash/steps/prompt_query/prompt_response_parsers.dart';
 import 'package:commanddash/steps/steps_utils.dart';
 
 class PromptQueryStep extends Step {
+  final String agentName;
   final String query;
   final List<Output> outputs;
   Map<String, Input> inputs;
   Map<String, Output> outputsUntilNow; //TODO:rename needed
   PromptQueryStep(
-      {required List<String>? outputIds,
+      {required this.agentName,
+      required List<String>? outputIds,
       required this.outputs,
       required this.query,
       required this.inputs,
@@ -25,12 +27,14 @@ class PromptQueryStep extends Step {
 
   factory PromptQueryStep.fromJson(
     Map<String, dynamic> json,
+    String agentName,
     String query,
     List<Output> outputs,
     Map<String, Input> inputs,
     Map<String, Output> outputsUntilNow,
   ) {
     return PromptQueryStep(
+      agentName: agentName,
       outputIds:
           (json['outputs'] as List<dynamic>).map((e) => e.toString()).toList(),
       outputs: outputs,
@@ -187,7 +191,8 @@ class PromptQueryStep extends Step {
         args: ProcessingFilesLoader(filesInvolved, message: 'Preparing Result')
             .toJson(),
         timeoutKind: TimeoutKind.sync);
-    final response = await generationRepository.getCompletion(prompt);
+    final response = await generationRepository.getCompletion(prompt, agentName,
+        command: 'untracked');
     final result = <Output>[];
     for (Output output in outputs) {
       if (output is CodeOutput) {

commanddash/pubspec.yaml

@@ -1,6 +1,6 @@
 name: commanddash
 description: Processing engine for CommandDash clients like the IDEs
-version: 0.1.4
+version: 0.1.5
 repository: https://github.com/Welltested-AI/commanddash
 
 environment:

dash_agent/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.4.0
+
+* **Github Data Object Update**: Updated github object to allow extraction of only code, issues, or both. When defining `Github` object via `WebDataObject.fromGithub`, you can pass list of `GithubExtract` enums containing all the intended data types that you want to be extracted and index. For example, passing `[GithubExtract.code, GithubExtract.issues]` will index both code and issues.
+* **Deep Crawling Web Page Support**: Added support for deep crawling the website to extract sites data without the need of providing explicit sitemap. You can use `WebDataObject.fromWebPage` along with base url of the website that you want to index. Additionally, you will also need to pass `deepCrawl` to `true` to enable deep crawling. By default the value is `false`, which means it will only index the shared url page only 
+
 ## 0.3.0
 
 * **Github Data Object Support**: Added support to extract Github repo data conviniently. You can use `WebDataObject.fromGithub` for indexing a specific repo code and issues now.

dash_agent/README.md

@@ -93,9 +93,10 @@ class DocsDataSource extends DataSource {
  /// URLs or sitemaps or even github repos in the object.
   @override
  List<WebDataObject> get webObjects =>
-     [WebDataObject.fromWebPage('https://sampleurl.com'), 
+     [WebDataObject.fromWebPage('https://sampleurl.com'),
+     WebDataObject.fromWebPage('https://sampleurl.com', deepCrawl: true), // to index all the pages of sampleurl.com
      WebDataObject.fromSiteMap('https://sampleurl.com/sitemap.xml'),
-     WebDataObject.fromGithub('https://github.com/user/repo', '<personal access token>', CodeFilter(pathRegex: '.*\.dart'),  IssueFilter(labels: ['bug']))];
+     WebDataObject.fromGithub('https://github.com/user/repo', '<personal access token>', codeFilter: CodeFilter(pathRegex: '.*\.dart'),  issueFilter: IssueFilter(labels: ['bug']))];
 }
 ```