From dc705ff646d94a30f2323a1dd18d2ac5c19dd7a4 Mon Sep 17 00:00:00 2001 From: Phodal Huang Date: Mon, 20 May 2024 20:01:35 +0800 Subject: [PATCH] docs(semantic): add Hyde Code and Keywords strategy documentation Added documentation for Hyde Code and Keywords strategy with detailed steps and code snippets. --- docs/development/prompt-example.md | 3 +- docs/semantic/hyde-code-strategy.md | 43 +++++++++++++++++++ docs/semantic/hyde-keywords-strategy.md | 40 +++++++++++++++++ docs/semantic/rag-strategy.md | 4 +- .../search-strategy/HydeCodeStrategy.ts | 5 +-- .../search-strategy/HydeKeywordsStrategy.ts | 26 +---------- .../search-strategy/_base/HydeStrategy.ts | 24 +++++++++++ 7 files changed, 114 insertions(+), 31 deletions(-) create mode 100644 docs/semantic/hyde-code-strategy.md create mode 100644 docs/semantic/hyde-keywords-strategy.md diff --git a/docs/development/prompt-example.md b/docs/development/prompt-example.md index 012dc69e..dc75b8a0 100644 --- a/docs/development/prompt-example.md +++ b/docs/development/prompt-example.md @@ -1,7 +1,8 @@ --- layout: default -title: Prompt Example +title: Unit Test Prompt Example nav_order: 999 +parent: Development --- ```devin diff --git a/docs/semantic/hyde-code-strategy.md b/docs/semantic/hyde-code-strategy.md new file mode 100644 index 00000000..a5bfa4c1 --- /dev/null +++ b/docs/semantic/hyde-code-strategy.md @@ -0,0 +1,43 @@ +--- +layout: default +title: Hyde Code Strategy +nav_order: 3 +parent: Semantic +--- + +Code file: HydeCodeStrategy.ts + +1. generate hyde doc code from the user query +2. retrieve code snippets by hyde code from the codebase +3. summarize the code snippets and return the result + +```typescript + channel.appendLine("=".repeat(80)); +channel.appendLine(`= Hyde Keywords Strategy: ${this.constructor.name} =`); +channel.appendLine("=".repeat(80)); + +this.step = HydeStep.Propose; +let documents = await this.generateDocument(); +let hydeCode = documents.content; + +this.step = HydeStep.Retrieve; +let chunks = await this.retrieveChunks(hydeCode); + +this.step = HydeStep.Evaluate; +let evaluateContext: KeywordEvaluateContext = { + step: this.step, + question: this.query, + code: chunks.map(item => item.text).join("\n"), + language: "" +}; + +if (chunks.length === 0) { + channel.appendLine("No code snippets found."); + return new StrategyOutput("", []); +} + +channel.appendLine("\n"); +channel.appendLine(" --- Summary --- "); +let evaluateIns = await PromptManager.getInstance().renderHydeTemplate(this.step, this.documentType, evaluateContext); +return new StrategyOutput(await executeIns(evaluateIns), chunks); +``` diff --git a/docs/semantic/hyde-keywords-strategy.md b/docs/semantic/hyde-keywords-strategy.md new file mode 100644 index 00000000..0cc7d6ab --- /dev/null +++ b/docs/semantic/hyde-keywords-strategy.md @@ -0,0 +1,40 @@ +--- +layout: default +title: Hyde Keywords Strategy +nav_order: 2 +parent: Semantic +--- + +Code file: HydeKeywordsStrategy.ts + +1. generate keywords from the user query +2. retrieve code snippets by query from the codebase +3. summarize the code snippets and return the result + +```typescript +this.step = HydeStep.Propose; +let documents = await this.generateDocument(); +let keywords = documents.content; + +this.step = HydeStep.Retrieve; +let queryTerm = this.createQueryTerm(keywords); +let chunkItems = await this.retrieveChunks(queryTerm); + +this.step = HydeStep.Evaluate; +let evaluateContext: KeywordEvaluateContext = { + step: this.step, + question: keywords.question, + code: chunkItems.map(item => item.text).join("\n"), + language: "" +}; + +if (chunkItems.length === 0) { + channel.appendLine("No code snippets found."); + return new StrategyOutput("", []); +} + +channel.appendLine("\n"); +channel.appendLine(" --- Summary --- "); +let evaluateIns = await PromptManager.getInstance().renderHydeTemplate(this.step, this.documentType, evaluateContext); +return new StrategyOutput(await executeIns(evaluateIns), chunkItems); +``` diff --git a/docs/semantic/rag-strategy.md b/docs/semantic/rag-strategy.md index 21b6edd1..7b3e7095 100644 --- a/docs/semantic/rag-strategy.md +++ b/docs/semantic/rag-strategy.md @@ -20,6 +20,6 @@ parent: Semantic ### Our Strategy -- HydeCodeStrategy -- HydeKeywordsStrategy +- HydeCodeStrategy reference to GitHub Copilot +- HydeKeywordsStrategy reference to Bloop: HydeDoc diff --git a/src/code-search/search-strategy/HydeCodeStrategy.ts b/src/code-search/search-strategy/HydeCodeStrategy.ts index d1f9fb3e..35ee4694 100644 --- a/src/code-search/search-strategy/HydeCodeStrategy.ts +++ b/src/code-search/search-strategy/HydeCodeStrategy.ts @@ -1,5 +1,5 @@ import { ChunkItem } from "../embedding/_base/Embedding"; -import { HydeQuery, HydeStrategy } from "./_base/HydeStrategy"; +import { executeIns, HydeQuery, HydeStrategy } from "./_base/HydeStrategy"; import { HydeDocument, HydeDocumentType } from "./_base/HydeDocument"; import { AutoDevExtension } from "../../AutoDevExtension"; import { ChatMessage } from "../../llm-provider/ChatMessage"; @@ -9,8 +9,7 @@ import { channel } from "../../channel"; import { LocalEmbeddingProvider } from "../embedding/LocalEmbeddingProvider"; import { DefaultRetrieval } from "../retrieval/DefaultRetrieval"; import { ContextItem, RetrieveOption } from "../retrieval/Retrieval"; -import { TextRange } from "../scope-graph/model/TextRange"; -import { executeIns, KeywordEvaluateContext, KeywordsProposeContext } from "./HydeKeywordsStrategy"; +import { KeywordEvaluateContext, KeywordsProposeContext } from "./HydeKeywordsStrategy"; import { StreamingMarkdownCodeBlock } from "../../markdown/StreamingMarkdownCodeBlock"; import { StrategyOutput } from "./_base/StrategyOutput"; diff --git a/src/code-search/search-strategy/HydeKeywordsStrategy.ts b/src/code-search/search-strategy/HydeKeywordsStrategy.ts index 8310ef50..04d19dcb 100644 --- a/src/code-search/search-strategy/HydeKeywordsStrategy.ts +++ b/src/code-search/search-strategy/HydeKeywordsStrategy.ts @@ -1,4 +1,4 @@ -import { HydeQuery, HydeStrategy } from "./_base/HydeStrategy"; +import { executeIns, HydeQuery, HydeStrategy } from "./_base/HydeStrategy"; import { ChunkItem } from "../embedding/_base/Embedding"; import { HydeDocument, HydeDocumentType } from "./_base/HydeDocument"; import { PromptManager } from "../../prompt-manage/PromptManager"; @@ -8,35 +8,11 @@ import { ChatMessage } from "../../llm-provider/ChatMessage"; import { HydeKeywords } from "./_base/HydeKeywords"; import { DefaultRetrieval } from "../retrieval/DefaultRetrieval"; import { AutoDevExtension } from "../../AutoDevExtension"; -import { CustomActionPrompt } from "../../prompt-manage/custom-action/CustomActionPrompt"; -import { AutoDevStatus, AutoDevStatusManager } from "../../editor/editor-api/AutoDevStatusManager"; -import { LlmProvider } from "../../llm-provider/LlmProvider"; import { channel } from "../../channel"; import { LocalEmbeddingProvider } from "../embedding/LocalEmbeddingProvider"; import { ContextItem, RetrieveOption } from "../retrieval/Retrieval"; import { StrategyOutput } from "./_base/StrategyOutput"; -export async function executeIns(instruction: string) { - console.log("\ninstruction: \n" + instruction); - let result = ""; - try { - let chatMessages = CustomActionPrompt.parseChatMessage(instruction); - AutoDevStatusManager.instance.setStatus(AutoDevStatus.InProgress); - let response = await LlmProvider.codeCompletion()._streamChat(chatMessages); - for await (let chatMessage of response) { - channel.append(chatMessage.content); - result += chatMessage.content; - } - - AutoDevStatusManager.instance.setStatus(AutoDevStatus.Done); - return result; - } catch (e) { - console.log("error:" + e); - AutoDevStatusManager.instance.setStatus(AutoDevStatus.Error); - return ""; - } -} - /** * The `HydeKeywordsStrategy` class is a part of the Hyde Strategy pattern and is used to generate keywords from a query. * These keywords are then used to retrieve similar code by symbols. diff --git a/src/code-search/search-strategy/_base/HydeStrategy.ts b/src/code-search/search-strategy/_base/HydeStrategy.ts index a70bc8d7..8f956fa5 100644 --- a/src/code-search/search-strategy/_base/HydeStrategy.ts +++ b/src/code-search/search-strategy/_base/HydeStrategy.ts @@ -1,6 +1,10 @@ import { ChunkItem, Embedding } from "../../embedding/_base/Embedding"; import { HydeDocument, HydeDocumentType } from "./HydeDocument"; import { StrategyOutput } from "./StrategyOutput"; +import { CustomActionPrompt } from "../../../prompt-manage/custom-action/CustomActionPrompt"; +import { AutoDevStatus, AutoDevStatusManager } from "../../../editor/editor-api/AutoDevStatusManager"; +import { LlmProvider } from "../../../llm-provider/LlmProvider"; +import { channel } from "../../../channel"; export type HydeQuery = string | RegExp | Embedding; @@ -64,3 +68,23 @@ export interface HydeStrategy { execute(): Promise; } +export async function executeIns(instruction: string) { + console.log("\ninstruction: \n" + instruction); + let result = ""; + try { + let chatMessages = CustomActionPrompt.parseChatMessage(instruction); + AutoDevStatusManager.instance.setStatus(AutoDevStatus.InProgress); + let response = await LlmProvider.codeCompletion()._streamChat(chatMessages); + for await (let chatMessage of response) { + channel.append(chatMessage.content); + result += chatMessage.content; + } + + AutoDevStatusManager.instance.setStatus(AutoDevStatus.Done); + return result; + } catch (e) { + console.log("error:" + e); + AutoDevStatusManager.instance.setStatus(AutoDevStatus.Error); + return ""; + } +} \ No newline at end of file