diff --git a/.github/actions/fetch_ai_json/action.yml b/.github/actions/fetch_ai_json/action.yml new file mode 100644 index 0000000..4f94753 --- /dev/null +++ b/.github/actions/fetch_ai_json/action.yml @@ -0,0 +1,46 @@ +name: Fetch AI JSON +inputs: + url: + description: 'URL of the native headers' + type: 'string' + required: true + product_type: + description: 'product type, available options: rtc, rtm' + type: 'string' + required: true + GH_TOKEN: + description: 'GitHub token' + type: 'string' + required: true + +runs: + using: composite + steps: + # Setup .npmrc file to publish to GitHub Packages + - uses: actions/setup-node@v3 + with: + node-version: '18.x' + + - name: Reconfigure git to use HTTP authentication + run: > + git config --global url."https://${{ inputs.GH_TOKEN }}@github.com/".insteadOf ssh://git@github.com/ + shell: bash + + - name: Fetch AI JSON from URL + run: | + yarn + mkdir -p ai/temp + curl -o ai/temp/differences.json ${{ inputs.url }} + yarn ts-node ai/index.ts ai/temp/differences.json configs/${{ inputs.product_type }}/ai/parameter_list.ts + shell: bash + + - name: Create pull request + uses: AgoraIO-Extensions/actions/.github/actions/pr@main + with: + github-token: ${{ inputs.GH_TOKEN }} + target-repo: ${{ github.workspace }} + target-branch: ${{ github.ref_name }} + target-branch-name-surffix: fetch-ai-json + pull-request-title: | + [AUTO] Fetch AI JSON with ${{ inputs.url }} + add-paths: configs/* diff --git a/.github/workflows/fetch_ai_json.yml b/.github/workflows/fetch_ai_json.yml index 4d3184c..17bbcf8 100644 --- a/.github/workflows/fetch_ai_json.yml +++ b/.github/workflows/fetch_ai_json.yml @@ -21,30 +21,8 @@ jobs: - uses: actions/checkout@v3 with: persist-credentials: false - # Setup .npmrc file to publish to GitHub Packages - - uses: actions/setup-node@v3 + - uses: ./.github/actions/fetch_ai_json with: - node-version: '18.x' - - - name: Reconfigure git to use HTTP authentication - run: > - git config --global url."https://${{ secrets.GH_TOKEN }}@github.com/".insteadOf ssh://git@github.com/ - - - name: Fetch AI JSON from URL - run: | - yarn - ts-node ai/index.ts ${{ inputs.url }} configs/${{ inputs.product_type }}/ai/parameter_list.ts - - - name: Create pull request - uses: AgoraIO-Extensions/actions/.github/actions/pr@main - with: - github-token: ${{ secrets.GH_TOKEN }} - target-repo: ${{ github.workspace }} - target-branch: ${{ github.ref_name }} - target-branch-name-surffix: fetch-ai-json - pull-request-title: | - [AUTO] Fetch AI JSON - pull-request-body: | - AI JSON source: - ${{ inputs.url }} - add-paths: configs/* + url: ${{ inputs.url }} + product_type: ${{ inputs.product_type }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 63d2677..f4a6235 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,10 +16,7 @@ jobs: # Setup .npmrc file to publish to GitHub Packages - uses: actions/setup-node@v3 with: - node-version: "18.x" - registry-url: "https://npm.pkg.github.com" - # Defaults to the user or organization that owns the workflow file - scope: "@agoraio-extensions" + node-version: '18.x' - name: Reconfigure git to use HTTP authentication run: > diff --git a/.github/workflows/update_headers.yml b/.github/workflows/update_headers.yml index e8f5bd6..e787c79 100644 --- a/.github/workflows/update_headers.yml +++ b/.github/workflows/update_headers.yml @@ -9,6 +9,10 @@ on: description: 'URL of the native headers' type: 'string' required: true + ai_json_url: + description: 'URL of the AI JSON' + type: 'string' + required: true version: description: 'native headers version' type: 'string' @@ -31,6 +35,14 @@ jobs: bash scripts/update_headers.sh ${{ inputs.url }} ${{ inputs.version }} ${{ inputs.product_type }} shell: bash + - name: Fetch AI JSON if ai_json_url is provided + if: ${{ inputs.ai_json_url != '' }} + uses: ./.github/actions/fetch_ai_json + with: + url: ${{ inputs.ai_json_url }} + product_type: ${{ inputs.product_type }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Create pull request uses: AgoraIO-Extensions/actions/.github/actions/pr@main with: diff --git a/.gitignore b/.gitignore index b46a779..bcb8cd2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .ccls-cache/ node_modules/ dist/ -package-lock.json \ No newline at end of file +package-lock.json +ai/temp/ \ No newline at end of file diff --git a/.npmrc b/.npmrc deleted file mode 100644 index 19f169a..0000000 --- a/.npmrc +++ /dev/null @@ -1 +0,0 @@ -@agoraio-extensions:registry=https://npm.pkg.github.com \ No newline at end of file diff --git a/.yarn/install-state.gz b/.yarn/install-state.gz index 668650d..1f06f32 100644 Binary files a/.yarn/install-state.gz and b/.yarn/install-state.gz differ diff --git a/ai/doc_ai_tool_processor.ts b/ai/doc_ai_tool_processor.ts new file mode 100644 index 0000000..41e39be --- /dev/null +++ b/ai/doc_ai_tool_processor.ts @@ -0,0 +1,94 @@ +import { exec } from 'child_process'; +import * as fs from 'fs'; + +interface DocParameter { + name: string; + type: string; + change_type: string; + old_value: string; + new_value: string; + is_output: boolean; + is_array: boolean; +} + +interface DocChanges { + diff: string[]; + parent_class: string; + language: string; + details: { + api_name: string; + api_signature: string; + change_type: string; + parameters: DocParameter[]; + }; +} + +export interface AIParameter { + parent_name: string; + parent_class: string; + is_output: boolean; + is_array: boolean; +} + +interface AIParameterObject { + [key: string]: AIParameter; +} + +export interface DocAIToolJson { + changes: { + api_changes: DocChanges[]; + struct_changes: DocChanges[]; + enum_changes: DocChanges[]; + }; +} + +export class DocAIToolJsonProcessor { + private data: DocAIToolJson[] | undefined; + + constructor(filepath: string) { + this.readJsonFromFile(filepath); + } + + private readJsonFromFile(filepath: string): void { + try { + const jsonData = fs.readFileSync(filepath, 'utf-8'); + this.data = JSON.parse(jsonData); + } catch (error) { + console.error('Error reading JSON file:', error); + } + } + + generateConfigFromDocAPIChanges(): AIParameterObject { + let output: AIParameterObject = {}; + if (!this.data) { + console.error('call readJsonFromFile() first.'); + return output; + } + this.data.map((docAIToolJson: DocAIToolJson) => { + docAIToolJson.changes.api_changes.map((item: DocChanges) => { + item.details.parameters.map((param: DocParameter) => { + let key = `${item.parent_class}:${item.details.api_name}.${param.name}@type`; + output[key] = { + parent_class: item.parent_class, + parent_name: item.details.api_name, + is_output: param.is_output, + is_array: param.is_array, + }; + }); + }); + }); + return output; + } + + saveConfigToFile(outputPath: string): void { + const config = this.generateConfigFromDocAPIChanges(); + const configString = `module.exports = ${JSON.stringify(config, null, 2)};`; + try { + fs.writeFileSync(outputPath, configString, 'utf-8'); + exec(`yarn eslint --fix ${outputPath}`); + console.log(`Configuration saved to ${outputPath}`); + } catch (error) { + console.error('Error writing configuration to file:', error); + } + } +} diff --git a/ai/index.ts b/ai/index.ts new file mode 100644 index 0000000..251b93e --- /dev/null +++ b/ai/index.ts @@ -0,0 +1,13 @@ +import { DocAIToolJsonProcessor } from './doc_ai_tool_processor'; + +const args = process.argv.slice(2); + +if (args.length !== 2) { + console.error('Usage: node script.js '); + process.exit(1); +} + +const [inputFilePath, outputFilePath] = args; + +const docAIToolJsonProcessor = new DocAIToolJsonProcessor(inputFilePath); +docAIToolJsonProcessor.saveConfigToFile(outputFilePath); diff --git a/configs/rtc/ai/parameter_list.ts b/configs/rtc/ai/parameter_list.ts new file mode 100644 index 0000000..f053ebf --- /dev/null +++ b/configs/rtc/ai/parameter_list.ts @@ -0,0 +1 @@ +module.exports = {}; diff --git a/package.json b/package.json index dad00bf..184a925 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "@agoraio-extensions/terra-core": "git@github.com:AgoraIO-Extensions/terra.git#head=main&workspace=terra-core", "lodash": "^4.17.21", "mustache": "^4.2.0", - "openai": "^4.29.1" + "openai": "^4.77.0" }, "devDependencies": { "@types/jest": "^29.5.1", @@ -40,6 +40,7 @@ "eslint-plugin-import": "^2.27.5", "eslint-plugin-jest": "^26.9.0", "eslint-plugin-prettier": "^4.0.0", + "https-proxy-agent": "^7.0.6", "jest": "^29.5.0", "prettier": "^2.0.5", "ts-jest": "^29.1.0", diff --git a/src/__tests__/parsers/cud_node_parser.test.ts b/src/__tests__/parsers/cud_node_parser.test.ts index c57fb2e..b97e818 100644 --- a/src/__tests__/parsers/cud_node_parser.test.ts +++ b/src/__tests__/parsers/cud_node_parser.test.ts @@ -13,11 +13,13 @@ import { MemberFunction, Struct, } from '@agoraio-extensions/cxx-parser'; + +import { ParseResult, TerraContext } from '@agoraio-extensions/terra-core'; + import CUDNodeParser, { CUDNodeParserArgs, isNodeMatched, } from '../../parsers/cud_node_parser'; -import { ParseResult, TerraContext } from '@agoraio-extensions/terra-core'; describe('CUDNodeParser', () => { it('isNodeMatched', () => { diff --git a/src/__tests__/parsers/override_node_parser.test.ts b/src/__tests__/parsers/override_node_parser.test.ts index 8f85b78..313c4f2 100644 --- a/src/__tests__/parsers/override_node_parser.test.ts +++ b/src/__tests__/parsers/override_node_parser.test.ts @@ -9,6 +9,7 @@ import { } from '@agoraio-extensions/cxx-parser'; import { TerraContext } from '@agoraio-extensions/terra-core'; + import { OverrideNodeParser, getOverrideNodeParserUserData, diff --git a/src/generators/custom_headers.ts b/src/generators/custom_headers.ts new file mode 100644 index 0000000..c1d5732 --- /dev/null +++ b/src/generators/custom_headers.ts @@ -0,0 +1,59 @@ +import { Diff } from '../utils/diff'; +import { askGPT } from '../utils/gpt_utils'; + +const prompt = ` +You are a C++ Code Inspector. Your task is to rename within some given C++ methods. +You should reply shortly and no need to explain the code. +You should provide all the methods in the same reply. +The first method is no need to change, but the left methods need to be renamed. +Given method: +\`\`\`c++ +{{ METHOD_SOURCE }} +\`\`\` +`; + +let methodSource = ` + virtual int joinChannel(const char* token, const char* channelId, const char* info, uid_t uid) = 0; + virtual int joinChannel(const char* token, const char* channelId, uid_t uid, const ChannelMediaOptions& options) = 0; +`; + +const prompt2 = ` +You are a file diff tool. Your task is to compare two versions of a C++ code. +I will provide you a diff result of two versions of a C++ code that is from bash command \`diff -r -u -N\`. +Given diff result: +{{ DIFF_SOURCE }} + + +Now, you need to provide a summary of the changes between the two versions. +`; + +const old_version = 'rtc_4.4.0'; +const new_version = 'rtc_4.5.0'; + +const old_version_path = `headers/${old_version}/include`; +const new_version_path = `headers/${new_version}/include`; +const blackList = ['include/rte_base', 'include/internal']; + +const diffTool = new Diff(old_version_path, new_version_path, blackList); +diffTool.setOutputDirectory(`temp/${old_version}↔${new_version}`); +diffTool.run(); + +let promptWithMethod = prompt + .replace('{{ METHOD_SOURCE }}', methodSource) + .trim(); + +// let promptWithDiff = prompt +// .replace( +// '{{ DIFF_SOURCE }}', +// fs.readFileSync( +// `temp/${old_version}↔${new_version}/AgoraBase.h.diff`, +// 'utf8' +// ) +// ) +// .trim(); + +// (async () => { +// // let res = await askGPT(promptWithMethod); +// let res = await askGPT(promptWithDiff); +// console.log(res); +// })(); diff --git a/src/generators/prompt.ts b/src/generators/prompt.ts new file mode 100644 index 0000000..9259db9 --- /dev/null +++ b/src/generators/prompt.ts @@ -0,0 +1 @@ +export const systemPrompt = `You are a C++ Code Inspector. Your task is to rename within some given C++ methods.`; diff --git a/src/parsers/index.ts b/src/parsers/index.ts index c9d3828..b862bff 100644 --- a/src/parsers/index.ts +++ b/src/parsers/index.ts @@ -25,4 +25,5 @@ export type BaseParserArgs = { configFilePath?: string; defaultConfig?: any; ignoreDefaultConfig?: boolean; + useAI?: boolean; }; diff --git a/src/parsers/pointer_to_array_parser.ts b/src/parsers/pointer_to_array_parser.ts index e6b3b60..a1aac16 100644 --- a/src/parsers/pointer_to_array_parser.ts +++ b/src/parsers/pointer_to_array_parser.ts @@ -8,13 +8,17 @@ import { } from '@agoraio-extensions/cxx-parser'; import { ParseResult, TerraContext } from '@agoraio-extensions/terra-core'; +import { AIParameter } from '../../ai/doc_ai_tool_processor'; + import { getConfigs } from '../utils/parser_utils'; import { BaseParserArgs } from './index'; +const AIConfigMethodParameters = require('../../configs/rtc/ai/method_parameters.ts'); const defaultConfig = require('../../configs/rtc/pointer_to_array'); function markArray( + args: BaseParserArgs, nodes: (Variable | MemberVariable)[], parentNode: CXXTerraNode, name_configs: string[], @@ -34,6 +38,16 @@ function markArray( return; } + if (args.useAI) { + let _config: AIParameter = + AIConfigMethodParameters[ + `${node.parent?.parent?.name}:${node.parent?.name}.${node.name}` + ]; + if (_config?.parent_name) { + node.type.kind = SimpleTypeKind.array_t; + } + } + if ( node.parent?.__TYPE === CXXTYPE.MemberFunction && node.__TYPE === CXXTYPE.Variable && @@ -86,6 +100,7 @@ export function PointerToArrayParser( file.nodes.forEach((node) => { if (node.__TYPE === CXXTYPE.Struct) { markArray( + args, node.asStruct().member_variables, node, name_configs, @@ -93,6 +108,7 @@ export function PointerToArrayParser( ); } else if (node.__TYPE === CXXTYPE.Clazz) { markArray( + args, node.asClazz().member_variables, node, name_configs, @@ -100,6 +116,7 @@ export function PointerToArrayParser( ); node.asClazz().methods.forEach((method) => { markArray( + args, method.parameters, method, name_configs, diff --git a/src/parsers/return_type_parser.ts b/src/parsers/return_type_parser.ts index 42a8a93..1a747cb 100644 --- a/src/parsers/return_type_parser.ts +++ b/src/parsers/return_type_parser.ts @@ -7,11 +7,14 @@ import { } from '@agoraio-extensions/cxx-parser'; import { ParseResult, TerraContext } from '@agoraio-extensions/terra-core'; +import { AIParameter } from '../../ai/doc_ai_tool_processor'; + import { isOutputVariable } from '../utils'; import { getConfigs } from '../utils/parser_utils'; import { BaseParserArgs } from './index'; +const AIConfigMethodParameters = require('../../configs/rtc/ai/method_parameters.ts'); const defaultConfig = require('../../configs/rtc/fixed_return_type_list.ts'); export type ReturnTypeParserArgs = BaseParserArgs & { @@ -97,6 +100,15 @@ export function ReturnTypeParser( }; } } + if (args.useAI) { + let config: AIParameter = + AIConfigMethodParameters[ + `${param.parent?.parent?.name}:${param.parent?.name}.${param.name}` + ]; + if (config && config.parent_name) { + param.is_output = config.is_output; + } + } } } } diff --git a/src/renderers/iris_doc_renderer.ts b/src/renderers/iris_doc_renderer.ts index d1e0277..1d597b4 100644 --- a/src/renderers/iris_doc_renderer.ts +++ b/src/renderers/iris_doc_renderer.ts @@ -1,8 +1,8 @@ import { strict as assert } from 'assert'; import { execSync } from 'child_process'; import * as fs from 'fs'; -import path from 'path'; import * as os from 'os'; +import path from 'path'; import { ParseResult, diff --git a/src/renderers/pointer_marker_gpt_renderer.ts b/src/renderers/pointer_marker_gpt_renderer.ts index 52c5f04..d4fd849 100644 --- a/src/renderers/pointer_marker_gpt_renderer.ts +++ b/src/renderers/pointer_marker_gpt_renderer.ts @@ -1,28 +1,36 @@ import { execSync } from 'child_process'; import * as fs from 'fs'; import path from 'path'; -import _ from 'lodash'; import { - ParseResult, - RenderResult, - TerraContext, + CXXFile, + SimpleTypeKind, + Struct, +} from '@agoraio-extensions/cxx-parser'; +import { + ParseResult, + RenderResult, + TerraContext, } from '@agoraio-extensions/terra-core'; +import _ from 'lodash'; + +import { + PointerArrayNameMapping, + PointerMarkerParserConfigMarker, +} from '../parsers/pointer_marker_parser'; import { askGPT } from '../utils/gpt_utils'; -import { PointerArrayNameMapping, PointerMarkerParserConfigMarker } from '../parsers/pointer_marker_parser'; -import { CXXFile, SimpleTypeKind, Struct } from '@agoraio-extensions/cxx-parser'; export interface PointerMarkerGPTRenderer { - configPath?: string + configPath?: string; } export function PointerMarkerGPTRenderer( - terraContext: TerraContext, - args?: PointerMarkerGPTRenderer, - parseResult?: ParseResult + terraContext: TerraContext, + args?: PointerMarkerGPTRenderer, + parseResult?: ParseResult ): RenderResult[] { - processGPT(parseResult!, args?.configPath); - return []; + processGPT(parseResult!, args?.configPath); + return []; } const prompt = ` @@ -37,61 +45,72 @@ Given struct: \`\`\` `; -const defualtConfigPath = path.resolve(`${__dirname}/../../configs/rtc/pointer_marker.config.ts`); +const defualtConfigPath = path.resolve( + `${__dirname}/../../configs/rtc/pointer_marker.config.ts` +); async function processGPT(parseResult: ParseResult, configPath?: string) { - let originalConfigPath = configPath ?? defualtConfigPath; - let originalMarkers = require(originalConfigPath).markers as PointerMarkerParserConfigMarker[]; - - let structs = parseResult.nodes - .flatMap((node) => (node as CXXFile).nodes) - .filter((node) => node.isStruct()) - .filter((node) => { - return node.asStruct() - .member_variables - .find((member) => member.type.kind === SimpleTypeKind.pointer_t) !== undefined; - }); + let originalConfigPath = configPath ?? defualtConfigPath; + let originalMarkers = require(originalConfigPath) + .markers as PointerMarkerParserConfigMarker[]; + + let structs = parseResult.nodes + .flatMap((node) => (node as CXXFile).nodes) + .filter((node) => node.isStruct()) + .filter((node) => { + return ( + node + .asStruct() + .member_variables.find( + (member) => member.type.kind === SimpleTypeKind.pointer_t + ) !== undefined + ); + }); - let markers: string[] = []; - - for (let st of structs) { - let struct = st.asStruct(); - let structSource = structToSource(struct); - let promptWithStruct = prompt - .replace('{{ STRUCT_NAME }}', struct.name) - .replace('{{ STRUCT_SOURCE }}', structSource) - .trim(); - let res = await askGPT(promptWithStruct); - if (res.length > 0) { - let jsonArray = Object.values(JSON.parse(res)); - if (jsonArray.length === 0) { - continue; - } - - // let newJsonArray = jsonArray as PointerArrayNameMapping[]; - let newJsonArray: PointerArrayNameMapping[] = []; - let originalMarker = originalMarkers.find((entry: any) => _.isMatch(struct, entry.node)); - if (originalMarker) { - for (let om of (jsonArray as PointerArrayNameMapping[])) { - if (om.lengthName.length === 0) { - continue; - } - let found = originalMarker.pointerArrayNameMappings?.find((entry) => entry.ptrName === om.ptrName); - // Only add the ptrName if it's not found in the original marker - let toAdd = found ? found : om; - newJsonArray.push(toAdd); - } - } - - if (newJsonArray.length > 0) { - let pointerArrayNameMappings = newJsonArray.map((entry: any) => { - return ` + let markers: string[] = []; + + for (let st of structs) { + let struct = st.asStruct(); + let structSource = structToSource(struct); + let promptWithStruct = prompt + .replace('{{ STRUCT_NAME }}', struct.name) + .replace('{{ STRUCT_SOURCE }}', structSource) + .trim(); + let res = await askGPT(promptWithStruct); + if (res.length > 0) { + let jsonArray = Object.values(JSON.parse(res)); + if (jsonArray.length === 0) { + continue; + } + + // let newJsonArray = jsonArray as PointerArrayNameMapping[]; + let newJsonArray: PointerArrayNameMapping[] = []; + let originalMarker = originalMarkers.find((entry: any) => + _.isMatch(struct, entry.node) + ); + if (originalMarker) { + for (let om of jsonArray as PointerArrayNameMapping[]) { + if (om.lengthName.length === 0) { + continue; + } + let found = originalMarker.pointerArrayNameMappings?.find( + (entry) => entry.ptrName === om.ptrName + ); + // Only add the ptrName if it's not found in the original marker + let toAdd = found ? found : om; + newJsonArray.push(toAdd); + } + } + + if (newJsonArray.length > 0) { + let pointerArrayNameMappings = newJsonArray.map((entry: any) => { + return ` { ptrName: "${entry.ptrName}", lengthName: "${entry.lengthName}", }`.trim(); - }); + }); - let marker = ` + let marker = ` { node: { __TYPE: CXXTYPE.Struct, @@ -102,13 +121,13 @@ async function processGPT(parseResult: ParseResult, configPath?: string) { ${pointerArrayNameMappings.join(',\n')} ], }`.trim(); - markers.push(marker); - } - } + markers.push(marker); + } } - console.log(markers); + } + console.log(markers); - let output = ` + let output = ` import { CXXTYPE } from "@agoraio-extensions/cxx-parser"; module.exports = { @@ -118,28 +137,32 @@ module.exports = { }; `.trim(); - fs.writeFileSync(originalConfigPath, output); + fs.writeFileSync(originalConfigPath, output); - // Reformat the file - execSync(`yarn prettier ${originalConfigPath} --write`, { - cwd: path.resolve(__dirname, '../../'), - }); + // Reformat the file + execSync(`yarn prettier ${originalConfigPath} --write`, { + cwd: path.resolve(__dirname, '../../'), + }); } function structToSource(struct: Struct): string { - let structName = struct.name; - let structContent = struct.member_variables.map((member) => { - let memberName = member.name; - let memberType = member.type.source; - let memberComment = member.comment.split('\n').map((line) => `/// ${line}`).join('\n'); - return ` + let structName = struct.name; + let structContent = struct.member_variables + .map((member) => { + let memberName = member.name; + let memberType = member.type.source; + let memberComment = member.comment + .split('\n') + .map((line) => `/// ${line}`) + .join('\n'); + return ` ${memberComment} ${memberType} ${memberName};`.trim(); - }).join('\n\n'); + }) + .join('\n\n'); - return ` + return ` struct ${structName} { ${structContent} };`.trim(); } - diff --git a/src/utils/diff.ts b/src/utils/diff.ts new file mode 100644 index 0000000..5dc7fd6 --- /dev/null +++ b/src/utils/diff.ts @@ -0,0 +1,174 @@ +import { execSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; + +// Interface to store differences +interface Difference { + filePath: string; // Path of the file with differences + diffs: string[]; // Store the output from the diff command as an array + diffsBlocks: string[]; // Store the output from the diff command as an array +} + +export class Diff { + private dirA: string; + private dirB: string; + private blackList: string[]; + private whiteList: string[]; + private outputDir?: string; // Optional output directory + + constructor( + dirA: string, + dirB: string, + blackList: string[] = [], + whiteList: string[] = [] + ) { + this.dirA = dirA; + this.dirB = dirB; + this.blackList = blackList; + this.whiteList = whiteList; + } + + // Set the output directory for diff results + public setOutputDirectory(outputDir: string): void { + this.outputDir = outputDir; + } + + // Clean the file path to remove unwanted parts + private cleanFilePath(filePath: string): string { + return filePath + .replace(/\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}$/, '') + .trim(); + } + + // Compare two directories using the diff command + private compareDirectories(): Difference[] { + const differences: Difference[] = []; + + console.log(`Comparing directories: ${this.dirA} vs ${this.dirB}`); + + try { + const result = execSync( + `diff -u -b -r --unified=50 ${this.dirA} ${this.dirB}`, + { + encoding: 'utf-8', + } + ); + + if (!result) { + console.log('No Differences found.'); + return differences; // No differences found + } + } catch (error: any) { + if (error.status === 1) { + console.log('Differences found.'); + const errorOutput = error.stdout || 'Differences found but no output.'; + const diffLines = errorOutput.split('\n'); + let currentFile: string = ''; + let currentDiff: string[] = []; + + diffLines.map((line: string, index: number) => { + if (line.startsWith('+++ ')) { + currentFile = line.substring(4).trim(); + currentFile = this.cleanFilePath(currentFile); + } else if (line.startsWith('diff -u -b')) { + if (currentFile && currentDiff.length > 0) { + differences[differences.length - 1].filePath = currentFile; + differences[differences.length - 1].diffs = currentDiff; + differences[differences.length - 1].diffsBlocks = currentDiff + .join('\n') + .split(/(?=@@ -)/) + .map((part) => + part.includes('@@ -') + ? part.split(/(?=@@ -)/).join('@@ -') + : part + ) + .filter(Boolean); + currentFile = ''; + currentDiff = []; + } + differences.push({ + filePath: currentFile, + diffs: currentDiff, + diffsBlocks: [], + }); + } else if (currentFile) { + currentDiff.push(line); + } + if (index === diffLines.length - 1) { + differences[differences.length - 1].filePath = currentFile; + differences[differences.length - 1].diffs = currentDiff; + differences[differences.length - 1].diffsBlocks = currentDiff + .join('\n') + .split(/(?=@@ -)/) + .map((part) => + part.includes('@@ -') + ? part.split(/(?=@@ -)/).join('@@ -') + : part + ) + .filter(Boolean); + } + }); + } else { + console.error(`Error executing diff: ${error.message}`); + console.error(`Command output: ${error.stdout}`); + console.error(`Error output: ${error.stderr}`); + } + } + + return differences; + } + + // Write differences to the specified output directory + private writeDifferencesToFile(differences: Difference[]): void { + if (!this.outputDir) { + console.warn('Output directory not set. Skipping writing differences.'); + return; + } + + if (fs.existsSync(this.outputDir)) { + fs.rmSync(this.outputDir!, { recursive: true }); + } + fs.mkdirSync(this.outputDir, { recursive: true }); + + differences.forEach((diff) => { + const outputFilePath = path.join( + this.outputDir!, + `${path.basename(diff.filePath)}.diff` + ); + const diffContent = diff.diffs.join('\n'); + fs.writeFileSync(outputFilePath, diffContent); + console.log(`Differences written to: ${outputFilePath}`); + }); + } + + public run(): Difference[] { + const relativePathA = path.relative(this.dirA, this.dirA); + const relativePathB = path.relative(this.dirB, this.dirB); + + if ( + this.blackList.includes(relativePathA) || + this.blackList.includes(relativePathB) + ) { + console.log(`Skipping comparison due to blacklist.`); + return []; + } + + if ( + this.whiteList.length > 0 && + !this.whiteList.includes(relativePathA) && + !this.whiteList.includes(relativePathB) + ) { + console.log(`Skipping comparison as not in whitelist.`); + return []; + } + + const differences = this.compareDirectories(); + + // Write differences to the output directory if set + if (this.outputDir) { + this.writeDifferencesToFile(differences); + } + debugger; + return differences; + } +} diff --git a/src/utils/gpt_utils.ts b/src/utils/gpt_utils.ts index 8da6d4a..b8d4540 100644 --- a/src/utils/gpt_utils.ts +++ b/src/utils/gpt_utils.ts @@ -1,48 +1,49 @@ -import OpenAI from "openai"; +import { HttpsProxyAgent } from 'https-proxy-agent'; +import openai, { ClientOptions } from 'openai'; -let _openAIClient: OpenAI | undefined = undefined; -function openAIClient(): OpenAI { - if (_openAIClient === undefined) { - _openAIClient = new OpenAI(); +let _openAIClient: openai | undefined = undefined; +function openAIClient(): openai { + if (_openAIClient === undefined) { + let configuration: ClientOptions = { + apiKey: process.env.OPENAI_API_KEY, + }; + if (process.env.environment !== 'production') { + configuration.httpAgent = new HttpsProxyAgent( + process.env.https_proxy ?? '' + ); } - return _openAIClient; + _openAIClient = new openai(configuration); + } + return _openAIClient; } -/// Make sure you add the following environment variables before you call this function -/// - OPENAI_API_KEY -/// - OPENAI_BASE_URL export async function askGPT(prompt: string): Promise { - console.log(`prompt:`); - console.log(prompt); + const completion: openai.Chat.ChatCompletion = + await openAIClient().chat.completions.create({ + model: 'gpt-4o', + messages: [{ role: 'user', content: prompt }], + top_p: 0.8, + }); - const completion: OpenAI.Chat.ChatCompletion = (await openAIClient().chat.completions.create({ - model: 'gpt-4', - messages: [{ role: 'user', content: prompt }], - temperature: 0.7, - })); - - let response: any | undefined = undefined; - try { - // We can only make a synchronous call inside terra at this time, but in this way the completions API returns a string, so we need to - // do some tricky thing here. - if (completion !== undefined && typeof completion === 'string') { - let completionStr = completion as string; - if (completionStr.length > 0) { - response = JSON.parse(completionStr); - } - } else if (typeof completion === 'object' && completion !== null) { - response = completion; - } else { - console.log('Param is neither a string nor an object'); - } - } catch (error) { - console.error(error); + let response: any | undefined = undefined; + try { + // We can only make a synchronous call inside terra at this time, but in this way the completions API returns a string, so we need to + // do some tricky thing here. + if (completion !== undefined && typeof completion === 'string') { + let completionStr = completion as string; + if (completionStr.length > 0) { + response = JSON.parse(completionStr); + } + } else if (typeof completion === 'object' && completion !== null) { + response = completion; + } else { + console.log('Param is neither a string nor an object'); } + } catch (error) { + console.error(error); + } - let res = response?.choices[0]?.message?.content ?? ''; - - console.log(`response:`); - console.log(res); + let res = response?.choices[0]?.message?.content ?? ''; - return res; -} \ No newline at end of file + return res; +} diff --git a/yarn.lock b/yarn.lock index 89421c6..2c03795 100644 --- a/yarn.lock +++ b/yarn.lock @@ -41,10 +41,11 @@ __metadata: eslint-plugin-import: "npm:^2.27.5" eslint-plugin-jest: "npm:^26.9.0" eslint-plugin-prettier: "npm:^4.0.0" + https-proxy-agent: "npm:^7.0.6" jest: "npm:^29.5.0" lodash: "npm:^4.17.21" mustache: "npm:^4.2.0" - openai: "npm:^4.29.1" + openai: "npm:^4.77.0" prettier: "npm:^2.0.5" ts-jest: "npm:^29.1.0" ts-node: "npm:^10.9.1" @@ -1308,6 +1309,13 @@ __metadata: languageName: node linkType: hard +"agent-base@npm:^7.1.2": + version: 7.1.3 + resolution: "agent-base@npm:7.1.3" + checksum: 6192b580c5b1d8fb399b9c62bf8343d76654c2dd62afcb9a52b2cf44a8b6ace1e3b704d3fe3547d91555c857d3df02603341ff2cb961b9cfe2b12f9f3c38ee11 + languageName: node + linkType: hard + "agentkeepalive@npm:^4.2.1": version: 4.5.0 resolution: "agentkeepalive@npm:4.5.0" @@ -3007,6 +3015,16 @@ __metadata: languageName: node linkType: hard +"https-proxy-agent@npm:^7.0.6": + version: 7.0.6 + resolution: "https-proxy-agent@npm:7.0.6" + dependencies: + agent-base: "npm:^7.1.2" + debug: "npm:4" + checksum: f729219bc735edb621fa30e6e84e60ee5d00802b8247aac0d7b79b0bd6d4b3294737a337b93b86a0bd9e68099d031858a39260c976dc14cdbba238ba1f8779ac + languageName: node + linkType: hard + "human-signals@npm:^2.1.0": version: 2.1.0 resolution: "human-signals@npm:2.1.0" @@ -4511,9 +4529,9 @@ __metadata: languageName: node linkType: hard -"openai@npm:^4.29.1": - version: 4.76.0 - resolution: "openai@npm:4.76.0" +"openai@npm:^4.77.0": + version: 4.77.0 + resolution: "openai@npm:4.77.0" dependencies: "@types/node": "npm:^18.11.18" "@types/node-fetch": "npm:^2.6.4" @@ -4529,7 +4547,7 @@ __metadata: optional: true bin: openai: bin/cli - checksum: f0b53906ba72e6d21405353f26ae3dd4327f3cbe212787fac321089772fccaaabfcd699abb8632e19c1eb536bd31e9fff3d1f90f7de693d86ae9624c04a9d74b + checksum: 438e5acbcdc592ff192f294e936c10a8b71edf898b53afacb937da45f8d4e221e041bfcc84d6174c8dcb9ed4080b32760f8d94de1fcec7ab889046f1e1173f68 languageName: node linkType: hard