From abaebbf806636eb37fe888ea34404bebbf574584 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 8 Oct 2024 14:36:14 -0400 Subject: [PATCH 01/17] feat: add codeium_api_key column to config table --- packages/api/db/schema.mts | 1 + .../0011_add_codeium_api_key_to_config.sql | 1 + packages/api/drizzle/meta/0011_snapshot.json | 260 ++++++++++++++++++ packages/api/drizzle/meta/_journal.json | 7 + 4 files changed, 269 insertions(+) create mode 100644 packages/api/drizzle/0011_add_codeium_api_key_to_config.sql create mode 100644 packages/api/drizzle/meta/0011_snapshot.json diff --git a/packages/api/db/schema.mts b/packages/api/db/schema.mts index 1bbf9783..bbb7b225 100644 --- a/packages/api/db/schema.mts +++ b/packages/api/db/schema.mts @@ -15,6 +15,7 @@ export const configs = sqliteTable('config', { aiProvider: text('ai_provider').notNull().default('openai'), aiModel: text('ai_model').default('gpt-4o'), aiBaseUrl: text('ai_base_url'), + codeiumApiKey: text('codeium_api_key'), // Null: unset. Email: subscribed. "dismissed": dismissed the dialog. subscriptionEmail: text('subscription_email'), }); diff --git a/packages/api/drizzle/0011_add_codeium_api_key_to_config.sql b/packages/api/drizzle/0011_add_codeium_api_key_to_config.sql new file mode 100644 index 00000000..7d275e83 --- /dev/null +++ b/packages/api/drizzle/0011_add_codeium_api_key_to_config.sql @@ -0,0 +1 @@ +ALTER TABLE `config` ADD `codeium_api_key` text; diff --git a/packages/api/drizzle/meta/0011_snapshot.json b/packages/api/drizzle/meta/0011_snapshot.json new file mode 100644 index 00000000..2d223b6c --- /dev/null +++ b/packages/api/drizzle/meta/0011_snapshot.json @@ -0,0 +1,260 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "a22c9bdd-1d54-448d-9c4a-5c3b2d7b7d60", + "prevId": "aeb418fb-06df-4fc2-8afc-f18d95014b46", + "tables": { + "apps": { + "name": "apps", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch())" + } + }, + "indexes": { + "apps_external_id_unique": { + "name": "apps_external_id_unique", + "columns": [ + "external_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "config": { + "name": "config", + "columns": { + "base_dir": { + "name": "base_dir", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_language": { + "name": "default_language", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'typescript'" + }, + "openai_api_key": { + "name": "openai_api_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anthropic_api_key": { + "name": "anthropic_api_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled_analytics": { + "name": "enabled_analytics", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "srcbook_installation_id": { + "name": "srcbook_installation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'k9nek54ld4r5881475jtrr5jns'" + }, + "ai_provider": { + "name": "ai_provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'openai'" + }, + "ai_model": { + "name": "ai_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'gpt-4o'" + }, + "ai_base_url": { + "name": "ai_base_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "codeium_api_key": { + "name": "codeium_api_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subscription_email": { + "name": "subscription_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "secrets": { + "name": "secrets", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "secrets_name_unique": { + "name": "secrets_name_unique", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "secrets_to_sessions": { + "name": "secrets_to_sessions", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret_id": { + "name": "secret_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "secrets_to_sessions_session_id_secret_id_unique": { + "name": "secrets_to_sessions_session_id_secret_id_unique", + "columns": [ + "session_id", + "secret_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "secrets_to_sessions_secret_id_secrets_id_fk": { + "name": "secrets_to_sessions_secret_id_secrets_id_fk", + "tableFrom": "secrets_to_sessions", + "tableTo": "secrets", + "columnsFrom": [ + "secret_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/api/drizzle/meta/_journal.json b/packages/api/drizzle/meta/_journal.json index 73cc4049..ef9ed6bf 100644 --- a/packages/api/drizzle/meta/_journal.json +++ b/packages/api/drizzle/meta/_journal.json @@ -78,6 +78,13 @@ "when": 1726808187994, "tag": "0010_create_apps", "breakpoints": true + }, + { + "idx": 11, + "version": "6", + "when": 1728410206740, + "tag": "0011_add_codeium_api_key_to_config", + "breakpoints": true } ] } \ No newline at end of file From aa6868576f26e580a6ab0d17a265b36a5d752aa9 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 8 Oct 2024 14:36:44 -0400 Subject: [PATCH 02/17] feat: add MVP of codeium oauth process --- packages/web/src/components/use-settings.tsx | 7 +- packages/web/src/lib/server.ts | 1 + packages/web/src/main.tsx | 6 ++ .../src/routes/settings-codeium-callback.tsx | 76 +++++++++++++++++++ packages/web/src/routes/settings.tsx | 50 +++++++++++- packages/web/src/types.ts | 1 + 6 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 packages/web/src/routes/settings-codeium-callback.tsx diff --git a/packages/web/src/components/use-settings.tsx b/packages/web/src/components/use-settings.tsx index 711838b4..78389388 100644 --- a/packages/web/src/components/use-settings.tsx +++ b/packages/web/src/components/use-settings.tsx @@ -24,7 +24,12 @@ export function SettingsProvider({ config, children }: ProviderPropsType) { const updateConfig = async (newConfig: Partial) => { // Filter out null values and convert back to an object const changeSet = Object.fromEntries( - Object.entries(newConfig).filter(([_, value]) => value !== null), + Object.entries(newConfig).filter(([key, value]) => { + if (key === 'codeiumApiKey') { + return true; + } + return value !== null; + }), ); await updateConfigServer(changeSet); diff --git a/packages/web/src/lib/server.ts b/packages/web/src/lib/server.ts index 5098b165..0c953e1c 100644 --- a/packages/web/src/lib/server.ts +++ b/packages/web/src/lib/server.ts @@ -240,6 +240,7 @@ interface EditConfigRequestType { aiBaseUrl?: string; aiModel?: string; aiProvider?: AiProviderType; + codeiumApiKey?: string; subscriptionEmail?: string | null; } diff --git a/packages/web/src/main.tsx b/packages/web/src/main.tsx index 6031c9b4..32ea6f1c 100644 --- a/packages/web/src/main.tsx +++ b/packages/web/src/main.tsx @@ -8,6 +8,7 @@ import Home, { loader as homeLoader } from './routes/home'; import Apps from './routes/apps'; import Session from './routes/session'; import Settings from './routes/settings'; +import SettingsCodiumCallback from './routes/settings-codeium-callback'; import Secrets from './routes/secrets'; import ErrorPage from './error'; import { DragAndDropSrcmdModal } from './components/drag-and-drop-srcmd-modal'; @@ -49,6 +50,11 @@ const router = createBrowserRouter([ element: , errorElement: , }, + { + path: '/settings/codeium-callback', + element: , + errorElement: , + }, { path: '/', element: ( diff --git a/packages/web/src/routes/settings-codeium-callback.tsx b/packages/web/src/routes/settings-codeium-callback.tsx new file mode 100644 index 00000000..500261d8 --- /dev/null +++ b/packages/web/src/routes/settings-codeium-callback.tsx @@ -0,0 +1,76 @@ +import { useState } from 'react'; +import { useNavigate, useSearchParams } from 'react-router-dom'; +import useEffectOnce from '@/components/use-effect-once'; +import { useSettings } from '@/components/use-settings'; + +export async function exchangeCodeiumAccessTokenForApiKey(accessToken: string): Promise<{ + api_key: string; + name: string; +}> { + // from: https://github.com/Exafunction/codeium.vim/blob/d85a85ca7e12967db22b00fd5e9f1a095bb47c96/autoload/codeium/command.vim#L90 + const response = await fetch('https://api.codeium.com/register_user/', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ firebase_id_token: accessToken }), + }); + + if (!response.ok) { + console.error(response); + throw new Error(`Error exchanging codeium access token for api key: ${response.status} ${await response.text()}`); + } + + return response.json(); +} + +function SettingsCodeiumCallback() { + const [status, setStatus] = useState< + 'loading' | 'already_set' | 'no_access_token' | 'token_exchange_error' + >('loading'); + const { codeiumApiKey, updateConfig } = useSettings(); + + const navigate = useNavigate(); + const [queryParams] = useSearchParams(); + + useEffectOnce(() => { + if (codeiumApiKey) { + setStatus("already_set"); + return; + } + const accessToken = queryParams.get('access_token'); + if (!accessToken) { + setStatus("no_access_token"); + return; + } + + exchangeCodeiumAccessTokenForApiKey(accessToken).then(async response => { + const apiKey = response.api_key; + await updateConfig({ codeiumApiKey: apiKey }); + + navigate('/settings'); + }).catch(err => { + console.error(err); + setStatus("token_exchange_error"); + }); + }); + + switch (status) { + case "loading": + return ( +
Loading...
+ ); + case "already_set": + return ( +
Codeium credentials already set!
+ ); + case "no_access_token": + return ( +
No access_token query parameter found!
+ ); + case "token_exchange_error": + return ( +
Error exchanging codeium access token for api key!
+ ); + } +}; + +export default SettingsCodeiumCallback; diff --git a/packages/web/src/routes/settings.tsx b/packages/web/src/routes/settings.tsx index b8272021..1657f816 100644 --- a/packages/web/src/routes/settings.tsx +++ b/packages/web/src/routes/settings.tsx @@ -1,5 +1,6 @@ import { useEffect, useState } from 'react'; -import { CircleCheck, Loader2, CircleX } from 'lucide-react'; +import { CircleCheck, Loader2, CircleX, EyeIcon, EyeOffIcon } from 'lucide-react'; +import { Link } from 'react-router-dom'; import { aiHealthcheck, subscribeToMailingList } from '@/lib/server'; import { useSettings } from '@/components/use-settings'; import { AiProviderType, getDefaultModel, type CodeLanguageType } from '@srcbook/shared'; @@ -21,6 +22,7 @@ function Settings() { aiProvider, aiModel, aiBaseUrl, + codeiumApiKey, openaiKey: configOpenaiKey, anthropicKey: configAnthropicKey, updateConfig: updateConfigContext, @@ -35,6 +37,7 @@ function Settings() { const [model, setModel] = useState(aiModel); const [baseUrl, setBaseUrl] = useState(aiBaseUrl || ''); const [email, setEmail] = useState(isSubscribed ? subscriptionEmail : ''); + const [codeiumApiKeyVisible, setCodeiumApiKeyVisible] = useState(false); const updateDefaultLanguage = (value: CodeLanguageType) => { updateConfigContext({ defaultLanguage: value }); @@ -81,6 +84,8 @@ function Settings() { } }; + const codeiumCallbackUrl = `${window.location.href}/codeium-callback`; + return (

Settings

@@ -206,6 +211,49 @@ function Settings() { )}
+
+

AI Autocomplete

+
+ {codeiumApiKey ? ( +
+
Codeium API Key:
+
+ + + +
+
+ ) : ( +
+ +
+ )} +
+

Get product updates

diff --git a/packages/web/src/types.ts b/packages/web/src/types.ts index 7d23392d..a89526ef 100644 --- a/packages/web/src/types.ts +++ b/packages/web/src/types.ts @@ -15,6 +15,7 @@ export type SettingsType = { aiProvider: AiProviderType; aiModel: string; aiBaseUrl?: string | null; + codeiumApiKey?: string | null; subscriptionEmail?: string | null; }; From 0a191be94a2b722e8a43dd219ffad0c83062a7a2 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 3 Oct 2024 16:33:19 -0400 Subject: [PATCH 03/17] feat: add hacked together demo for codium ai autocomplete --- .../api/ai/autocomplete/codeium_common.proto | 166 +++++++++++++++++ packages/api/ai/autocomplete/index.mts | 73 ++++++++ .../api/ai/autocomplete/language_server.proto | 169 ++++++++++++++++++ packages/api/package.json | 2 + packages/api/server/http.mts | 14 ++ packages/shared/index.mts | 1 + packages/shared/src/types/ai-autocomplete.mts | 49 +++++ packages/web/package.json | 9 +- packages/web/src/components/cells/code.tsx | 25 ++- packages/web/src/lib/server.ts | 15 ++ pnpm-lock.yaml | 97 ++++++++++ 11 files changed, 615 insertions(+), 5 deletions(-) create mode 100644 packages/api/ai/autocomplete/codeium_common.proto create mode 100644 packages/api/ai/autocomplete/index.mts create mode 100644 packages/api/ai/autocomplete/language_server.proto create mode 100644 packages/shared/src/types/ai-autocomplete.mts diff --git a/packages/api/ai/autocomplete/codeium_common.proto b/packages/api/ai/autocomplete/codeium_common.proto new file mode 100644 index 00000000..7c06bf44 --- /dev/null +++ b/packages/api/ai/autocomplete/codeium_common.proto @@ -0,0 +1,166 @@ +// Copyright Exafunction, Inc. + +syntax = "proto3"; + +package exa.codeium_common_pb; + +/* import "google/protobuf/duration.proto"; */ +/* import "google/protobuf/timestamp.proto"; */ +/* import "validate/validate.proto"; */ + +option go_package = "github.com/Exafunction/Exafunction/exa/codeium_common_pb"; + +enum ExperimentKey { + UNSPECIFIED = 0; + JUPYTER_FORMAT = 77; +} + +// Next ID: 12, Previous field: entropy. +message Completion { + string completion_id = 1; + string text = 2; + string prefix = 3; + string stop = 4; + double score = 5; + repeated uint64 tokens = 6; + repeated string decoded_tokens = 7; + repeated double probabilities = 8; + repeated double adjusted_probabilities = 9; + uint64 generated_length = 10; +} + +// Authentication source for users on the cloud service. +enum AuthSource { + AUTH_SOURCE_CODEIUM = 0; +} + +// Next ID: 15, Previous field: url. +message Metadata { + string ide_name = 1 /* [(validate.rules).string.min_len = 1] */; + string ide_version = 7 /* [(validate.rules).string.min_len = 1] */; + string extension_name = 12; + string extension_version = 2 /* [(validate.rules).string.min_len = 1] */; + string api_key = 3 /* [(validate.rules).string.uuid = true] */; + // Regex derived from https://stackoverflow.com/a/48300605. + // TODO(prem): Should this be mandatory? + string locale = 4 /* [(validate.rules).string = { + ignore_empty: true, + pattern: "^[A-Za-z]{2,4}([_-][A-Za-z]{4})?([_-]([A-Za-z]{2}|[0-9]{3}))?$" + }] */; + // UID identifying a single session for the given user. + string session_id = 10; + + // Used purely in language server to cancel in flight requests. + // If request_id is 0, then the request is not cancelable. + // This should be a strictly monotonically increasing number + // for the duration of a session. + uint64 request_id = 9; + + // Browser-specific information. + string user_agent = 13; + string url = 14 /* [(validate.rules).string = { + ignore_empty: true, + uri: true + }] */; + + // Authentication source information. + AuthSource auth_source = 15; +} + +// Next ID: 3, Previous field: insert_spaces. +message EditorOptions { + uint64 tab_size = 1 /* [(validate.rules).uint64.gt = 0] */; + bool insert_spaces = 2; +} + +message Event { + EventType event_type = 1; + string event_json = 2; + int64 timestamp_unix_ms = 3; +} + +enum EventType { + EVENT_TYPE_UNSPECIFIED = 0; + EVENT_TYPE_ENABLE_CODEIUM = 1; + EVENT_TYPE_DISABLE_CODEIUM = 2; + EVENT_TYPE_SHOW_PREVIOUS_COMPLETION = 3; + EVENT_TYPE_SHOW_NEXT_COMPLETION = 4; +} + +enum CompletionSource { + COMPLETION_SOURCE_UNSPECIFIED = 0; + COMPLETION_SOURCE_TYPING_AS_SUGGESTED = 1; + COMPLETION_SOURCE_CACHE = 2; + COMPLETION_SOURCE_NETWORK = 3; +} + +// Every time this list is updated, we should be redeploying the API server +// since it uses the string representation for BQ. +enum Language { + LANGUAGE_UNSPECIFIED = 0; + LANGUAGE_C = 1; + LANGUAGE_CLOJURE = 2; + LANGUAGE_COFFEESCRIPT = 3; + LANGUAGE_CPP = 4; + LANGUAGE_CSHARP = 5; + LANGUAGE_CSS = 6; + LANGUAGE_CUDACPP = 7; + LANGUAGE_DOCKERFILE = 8; + LANGUAGE_GO = 9; + LANGUAGE_GROOVY = 10; + LANGUAGE_HANDLEBARS = 11; + LANGUAGE_HASKELL = 12; + LANGUAGE_HCL = 13; + LANGUAGE_HTML = 14; + LANGUAGE_INI = 15; + LANGUAGE_JAVA = 16; + LANGUAGE_JAVASCRIPT = 17; + LANGUAGE_JSON = 18; + LANGUAGE_JULIA = 19; + LANGUAGE_KOTLIN = 20; + LANGUAGE_LATEX = 21; + LANGUAGE_LESS = 22; + LANGUAGE_LUA = 23; + LANGUAGE_MAKEFILE = 24; + LANGUAGE_MARKDOWN = 25; + LANGUAGE_OBJECTIVEC = 26; + LANGUAGE_OBJECTIVECPP = 27; + LANGUAGE_PERL = 28; + LANGUAGE_PHP = 29; + LANGUAGE_PLAINTEXT = 30; + LANGUAGE_PROTOBUF = 31; + LANGUAGE_PBTXT = 32; + LANGUAGE_PYTHON = 33; + LANGUAGE_R = 34; + LANGUAGE_RUBY = 35; + LANGUAGE_RUST = 36; + LANGUAGE_SASS = 37; + LANGUAGE_SCALA = 38; + LANGUAGE_SCSS = 39; + LANGUAGE_SHELL = 40; + LANGUAGE_SQL = 41; + LANGUAGE_STARLARK = 42; + LANGUAGE_SWIFT = 43; + LANGUAGE_TSX = 44; + LANGUAGE_TYPESCRIPT = 45; + LANGUAGE_VISUALBASIC = 46; + LANGUAGE_VUE = 47; + LANGUAGE_XML = 48; + LANGUAGE_XSL = 49; + LANGUAGE_YAML = 50; + LANGUAGE_SVELTE = 51; + LANGUAGE_TOML = 52; + LANGUAGE_DART = 53; + LANGUAGE_RST = 54; + LANGUAGE_OCAML = 55; + LANGUAGE_CMAKE = 56; + LANGUAGE_PASCAL = 57; + LANGUAGE_ELIXIR = 58; + LANGUAGE_FSHARP = 59; + LANGUAGE_LISP = 60; + LANGUAGE_MATLAB = 61; + LANGUAGE_POWERSHELL = 62; + LANGUAGE_SOLIDITY = 63; + LANGUAGE_ADA = 64; + LANGUAGE_OCAML_INTERFACE = 65; +} diff --git a/packages/api/ai/autocomplete/index.mts b/packages/api/ai/autocomplete/index.mts new file mode 100644 index 00000000..fa0337a9 --- /dev/null +++ b/packages/api/ai/autocomplete/index.mts @@ -0,0 +1,73 @@ +import path from 'node:path'; +import protobuf from 'protobufjs'; +import Long from 'long'; +import { type CodiumCompletionResult } from "@srcbook/shared"; + +// NOTE: this EDITOR_API_KEY value was just included as a raw string in +// @codeium/react-code-editor. This seems to not be a secret? +const EDITOR_API_KEY = 'd49954eb-cfba-4992-980f-d8fb37f0e942'; +const LANGUAGE_SERVER_PROTO_FILE_PATH = path.join(__dirname, "language_server.proto"); + +export async function runCodiumAiAutocomplete(source: string, cursorOffset: number): Promise { + const protos = await protobuf.load(LANGUAGE_SERVER_PROTO_FILE_PATH) + const GetCompletionsRequest = protos.lookupType("GetCompletionsRequest"); + const Metadata = protos.lookupType("Metadata"); + const DocumentInfo = protos.lookupType("Document"); + const EditorOptions = protos.lookupType("EditorOptions"); + const Language = protos.lookupEnum("Language"); + const GetCompletionsResponse = protos.lookupType("GetCompletionsResponse"); + + const sessionId = `react-editor-${crypto.randomUUID()}`; + + const payload = { + otherDocuments: [], + metadata: Metadata.create({ + ideName: 'web', + extensionVersion: '1.0.12', + apiKey: 'd49954eb-cfba-4992-980f-d8fb37f0e942', + ideVersion: 'unknown', + extensionName: '@codeium/react-code-editor', + sessionId, + }), + document: DocumentInfo.create({ + text: source, + editorLanguage: 'javascript', + language: Language.getOption("JAVASCRIPT"), + cursorOffset: Long.fromValue(cursorOffset), + lineEnding: '\n', + }), + editorOptions: EditorOptions.create({ + tabSize: Long.fromValue(4), + insertSpaces: true + }), + }; + + // const verified = GetCompletionsRequest.verify(payload); + // console.log('VERIFIED?', verified); + + // console.log('REQUEST:', payload); + + const requestData = GetCompletionsRequest.create(payload); + const buffer = GetCompletionsRequest.encode(requestData).finish(); + + const response = await fetch('https://web-backend.codeium.com/exa.language_server_pb.LanguageServerService/GetCompletions', { + method: 'POST', + body: buffer, + headers: { + 'Connect-Protocol-Version': '1', + 'Content-Type': 'application/proto', + Authorization: `Basic ${EDITOR_API_KEY}-${sessionId}`, + }, + }); + // console.log('RESPONSE:', response.status); + + const responseBodyBytes = new Uint8Array(await response.arrayBuffer()); + const responseBody = GetCompletionsResponse.decode(responseBodyBytes); + + // console.log('RESPONSE COMPLETIONS:'); + // for (const item of responseBody.completionItems) { + // console.log(item.completion.text); + // } + + return responseBody; +} diff --git a/packages/api/ai/autocomplete/language_server.proto b/packages/api/ai/autocomplete/language_server.proto new file mode 100644 index 00000000..84fcb518 --- /dev/null +++ b/packages/api/ai/autocomplete/language_server.proto @@ -0,0 +1,169 @@ +// Copyright Exafunction, Inc. + +syntax = "proto3"; + +package exa.language_server_pb; + +import "codeium_common.proto"; +// import "validate/validate.proto"; + +option go_package = "github.com/Exafunction/Exafunction/exa/language_server_pb"; + +service LanguageServerService { + rpc GetCompletions(GetCompletionsRequest) returns (GetCompletionsResponse) {} + rpc AcceptCompletion(AcceptCompletionRequest) returns (AcceptCompletionResponse) {} + rpc GetAuthToken(GetAuthTokenRequest) returns (GetAuthTokenResponse) {} +} + +message MultilineConfig { + // Multiline model threshold. 0-1, higher = more single line, lower = more multiline, + // 0.0 = only_multiline, default is 0.5 + float threshold = 1; +} + +// Next ID: 9, Previous field: disable_cache. +message GetCompletionsRequest { + codeium_common_pb.Metadata metadata = 1 /* [(validate.rules).message.required = true] */; + Document document = 2 /* [(validate.rules).message.required = true] */; + codeium_common_pb.EditorOptions editor_options = 3 /* [(validate.rules).message.required = true] */; + repeated Document other_documents = 5; + ExperimentConfig experiment_config = 7; + + string model_name = 10; + MultilineConfig multiline_config = 13; +} + +// Next ID: 5, Previous field: latency_info. +message GetCompletionsResponse { + State state = 1; + repeated CompletionItem completion_items = 2; +} + +// Next ID: 3, Previous field: completion_id. +message AcceptCompletionRequest { + codeium_common_pb.Metadata metadata = 1 /* [(validate.rules).message.required = true] */; + string completion_id = 2; +} + +// Next ID: 1, Previous field: N/A. +message AcceptCompletionResponse {} + +// Next ID: 1, Previous field: N/A. +message GetAuthTokenRequest {} + +// Next ID: 3, Previous field: uuid. +message GetAuthTokenResponse { + string auth_token = 1; + string uuid = 2; +} + +/*****************************************************************************/ +/* Helper Messages */ +/*****************************************************************************/ + +message DocumentPosition { + // 0-indexed. Measured in UTF-8 bytes. + uint64 row = 1; + // 0-indexed. Measured in UTF-8 bytes. + uint64 col = 2; +} + +// Next ID: 9, Previous field: cursor_position. +message Document { + string absolute_path = 1; + // Path relative to the root of the workspace. + string relative_path = 2; + string text = 3; + // Language ID provided by the editor. + string editor_language = 4 /* [(validate.rules).string.min_len = 1] */; + // Language enum standardized across editors. + codeium_common_pb.Language language = 5; + // Measured in number of UTF-8 bytes. + uint64 cursor_offset = 6; + // May be present instead of cursor_offset. + DocumentPosition cursor_position = 8; + // \n or \r\n, if known. + string line_ending = 7 /* [(validate.rules).string = { + in: [ + "", + "\n", + "\r\n" + ] + }] */; +} + +message ExperimentConfig { + repeated codeium_common_pb.ExperimentKey force_enable_experiments = 1 /* [(validate.rules).repeated.unique = true] */; +} + +enum CodeiumState { + CODEIUM_STATE_UNSPECIFIED = 0; + CODEIUM_STATE_INACTIVE = 1; + CODEIUM_STATE_PROCESSING = 2; + CODEIUM_STATE_SUCCESS = 3; + CODEIUM_STATE_WARNING = 4; + CODEIUM_STATE_ERROR = 5; +} + +// Next ID: 3, Previous field: message. +message State { + CodeiumState state = 1; + string message = 2; +} + +enum LineType { + LINE_TYPE_UNSPECIFIED = 0; + LINE_TYPE_SINGLE = 1; + LINE_TYPE_MULTI = 2; +} + +// Next ID: 5, Previous field: end_position. +message Range { + uint64 start_offset = 1; + uint64 end_offset = 2; + DocumentPosition start_position = 3; + DocumentPosition end_position = 4; +} + +message Suffix { + // Text to insert after the cursor when accepting the completion. + string text = 1; + // Cursor position delta (as signed offset) from the end of the inserted + // completion (including the suffix). + int64 delta_cursor_offset = 2; +} + +enum CompletionPartType { + COMPLETION_PART_TYPE_UNSPECIFIED = 0; + // Single-line completion parts that appear within an existing line of text. + COMPLETION_PART_TYPE_INLINE = 1; + // Possibly multi-line completion parts that appear below an existing line of text. + COMPLETION_PART_TYPE_BLOCK = 2; + // Like COMPLETION_PART_TYPE_INLINE, but overwrites the existing text. + COMPLETION_PART_TYPE_INLINE_MASK = 3; +} + +// Represents a contiguous part of the completion text that is not +// already in the document. +// Next ID: 4, Previous field: prefix. +message CompletionPart { + string text = 1; + // Offset in the original document where the part starts. For block + // parts, this is always the end of the line before the block. + uint64 offset = 2; + CompletionPartType type = 3; + // The section of the original line that came before this part. Only valid for + // COMPLETION_PART_TYPE_INLINE. + string prefix = 4; + // In the case of COMPLETION_PART_TYPE_BLOCK, represents the line it is below. + uint64 line = 5; +} + +// Next ID: 9, Previous field: completion_parts. +message CompletionItem { + codeium_common_pb.Completion completion = 1; + Suffix suffix = 5; + Range range = 2; + codeium_common_pb.CompletionSource source = 3; + repeated CompletionPart completion_parts = 8; +} diff --git a/packages/api/package.json b/packages/api/package.json index d89945b6..064aaddb 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -31,8 +31,10 @@ "depcheck": "^1.4.7", "drizzle-orm": "^0.33.0", "express": "^4.20.0", + "long": "^5.2.3", "marked": "catalog:", "posthog-node": "^4.2.0", + "protobufjs": "^7.4.0", "ws": "catalog:", "zod": "catalog:" }, diff --git a/packages/api/server/http.mts b/packages/api/server/http.mts index 6a5096cc..51219b26 100644 --- a/packages/api/server/http.mts +++ b/packages/api/server/http.mts @@ -14,6 +14,7 @@ import { exportSrcmdText, } from '../session.mjs'; import { generateCells, generateSrcbook, healthcheck } from '../ai/generate.mjs'; +import { runCodiumAiAutocomplete } from '../ai/autocomplete/index.mjs'; import { getConfig, updateConfig, @@ -348,6 +349,19 @@ router.post('/feedback', cors(), async (req, res) => { return res.json({ success: result.ok }); }); +router.options('/ai-autocomplete', cors()); +router.post('/ai-autocomplete', cors(), async (req, res) => { + const { source, cursorOffset } = req.body; + let result; + try { + result = await runCodiumAiAutocomplete(source, cursorOffset); + } catch (err) { + console.error('Error running ai autocomplete:', err); + return res.json({ error: true }); + } + return res.json({ error: false, result }); +}); + type NpmSearchResult = { package: { name: string; diff --git a/packages/shared/index.mts b/packages/shared/index.mts index 61c47c40..6e39bc11 100644 --- a/packages/shared/index.mts +++ b/packages/shared/index.mts @@ -7,5 +7,6 @@ export * from './src/types/cells.mjs'; export * from './src/types/tsserver.mjs'; export * from './src/types/websockets.mjs'; export * from './src/types/secrets.mjs'; +export * from './src/types/ai-autocomplete.mjs'; export * from './src/utils.mjs'; export * from './src/ai.mjs'; diff --git a/packages/shared/src/types/ai-autocomplete.mts b/packages/shared/src/types/ai-autocomplete.mts new file mode 100644 index 00000000..8ff2bd38 --- /dev/null +++ b/packages/shared/src/types/ai-autocomplete.mts @@ -0,0 +1,49 @@ +export type CodiumCompletionItem = { + completion: { + completionId: string; + text: string; + prefix: string; + stop: string; + score: number; + tokens: Array; + decoded_tokens: Array; + probabilities: Array; + adjustedProbabilities: Array; + generatedLength: string; + }; + completionParts: Array<{ + text: string; + offset: string; + prefix: string; + type: ( + | "COMPLETION_PART_TYPE_UNSPECIFIED" + // Single-line completion parts that appear within an existing line of text. + | "COMPLETION_PART_TYPE_INLINE" + // Possibly multi-line completion parts that appear below an existing line of text. + | "COMPLETION_PART_TYPE_BLOCK" + // Like COMPLETION_PART_TYPE_INLINE, but overwrites the existing text. + | "COMPLETION_PART_TYPE_INLINE_MASK" + ); + }>; + range: { + endOffset: string; + endPosition: { row?: string; col?: string }; + startPosition: { row?: string; col?: string }; + }; +}; + +export type CodiumCompletionResult = { + completionItems?: Array; + state: { + state: ( + | "CODEIUM_STATE_UNSPECIFIED" + | "CODEIUM_STATE_INACTIVE" + | "CODEIUM_STATE_PROCESSING" + | "CODEIUM_STATE_SUCCESS" + | "CODEIUM_STATE_WARNING" + | "CODEIUM_STATE_ERROR" + ), + status: string; + }; +}; + diff --git a/packages/web/package.json b/packages/web/package.json index 979a78c9..56778dc2 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -16,17 +16,19 @@ "@codemirror/autocomplete": "^6.18.1", "@codemirror/lang-css": "^6.3.0", "@codemirror/lang-html": "^6.4.9", + "@codemirror/lang-javascript": "^6.2.2", "@codemirror/lang-json": "^6.0.1", "@codemirror/lang-markdown": "^6.2.5", - "@codemirror/lang-javascript": "^6.2.2", "@codemirror/lint": "^6.8.1", - "@srcbook/shared": "workspace:^", "@srcbook/components": "workspace:^", + "@srcbook/shared": "workspace:^", "@uiw/codemirror-themes": "^4.23.2", "@uiw/react-codemirror": "^4.23.2", "clsx": "^2.1.1", "codemirror": "^6.0.1", + "codemirror-copilot": "^0.0.7", "lucide-react": "^0.439.0", + "marked": "catalog:", "react": "^18.3.1", "react-dom": "^18.3.1", "react-dropzone": "^14.2.3", @@ -37,8 +39,7 @@ "tailwind-merge": "^2.5.2", "tailwindcss-animate": "^1.0.7", "use-debounce": "^10.0.3", - "zod": "catalog:", - "marked": "catalog:" + "zod": "catalog:" }, "devDependencies": { "@types/react": "^18.3.5", diff --git a/packages/web/src/components/cells/code.tsx b/packages/web/src/components/cells/code.tsx index 8f5206f8..89aede9b 100644 --- a/packages/web/src/components/cells/code.tsx +++ b/packages/web/src/components/cells/code.tsx @@ -18,7 +18,7 @@ import { SessionChannel } from '@/clients/websocket'; import { useCells } from '@srcbook/components/src/components/use-cell'; import { mapCMLocationToTsServer, mapTsServerLocationToCM } from './util'; import { toast } from 'sonner'; -import { getFileContent } from '@/lib/server'; +import { getFileContent, runCodiumAiAutocomplete } from '@/lib/server'; import { tsHover } from '@/components/cells/hover'; import { autocompletion } from '@codemirror/autocomplete'; import { type Diagnostic, linter } from '@codemirror/lint'; @@ -34,6 +34,7 @@ import CodeMirror, { } from '@uiw/react-codemirror'; import useTheme from '@srcbook/components/src/components/use-theme'; import { Dialog, DialogContent } from '@srcbook/components/src/components/ui/dialog'; +import { inlineCopilot } from "codemirror-copilot"; function tsLinter( cell: CodeCellType, @@ -379,6 +380,28 @@ export default function ControlledCodeCell(props: Props) { }), ); } + extensions.push( + inlineCopilot(async (prefix, suffix) => { + let response; + try { + response = await runCodiumAiAutocomplete(prefix+suffix, prefix.length-1); + } catch (err) { + console.error('Error fetching ai autocomplete suggestion:', err); + return ""; + } + + console.log('AUTOCOMPLETE RESPONSE:', response); + if (response.error) { + return ""; + } + + const completionItems = response.result.completionItems ?? []; + const mostLikelyCompletionScore = Math.min(...completionItems.map(item => item.completion.score)); + const mostLikelyCompletion = completionItems.find(item => item.completion.score === mostLikelyCompletionScore); + + return mostLikelyCompletion?.completionParts[0]?.text ?? ""; + }, DEBOUNCE_DELAY), + ); extensions.push( Prec.highest( EditorView.domEventHandlers({ diff --git a/packages/web/src/lib/server.ts b/packages/web/src/lib/server.ts index 0c953e1c..fb3d7043 100644 --- a/packages/web/src/lib/server.ts +++ b/packages/web/src/lib/server.ts @@ -4,6 +4,7 @@ import type { MarkdownCellType, CodeCellType, SecretWithAssociatedSessions, + CodiumCompletionResult, } from '@srcbook/shared'; import { SessionType, ExampleSrcbookType } from '@/types'; import SRCBOOK_CONFIG from '@/config'; @@ -23,6 +24,20 @@ export async function getFileContent(filename: string) { return await file_response.json(); } +export async function runCodiumAiAutocomplete(source: string, cursorOffset: number): Promise<{error: false, result: CodiumCompletionResult} | { error: true }> { + const file_response = await fetch(API_BASE_URL + '/ai-autocomplete', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + source, + cursorOffset, + }), + }); + return await file_response.json(); +} + interface CreateSrcbookRequestType { path: string; name: string; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d517507c..d647e2cd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,12 +78,18 @@ importers: express: specifier: ^4.20.0 version: 4.20.0 + long: + specifier: ^5.2.3 + version: 5.2.3 marked: specifier: 'catalog:' version: 14.1.2 posthog-node: specifier: ^4.2.0 version: 4.2.0 + protobufjs: + specifier: ^7.4.0 + version: 7.4.0 ws: specifier: 'catalog:' version: 8.18.0 @@ -319,6 +325,9 @@ importers: codemirror: specifier: ^6.0.1 version: 6.0.1(@lezer/common@1.2.1) + codemirror-copilot: + specifier: ^0.0.7 + version: 0.0.7(@codemirror/state@6.4.1)(@codemirror/view@6.33.0) lucide-react: specifier: ^0.439.0 version: 0.439.0(react@18.3.1) @@ -1240,6 +1249,36 @@ packages: resolution: {integrity: sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.4': + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + + '@protobufjs/eventemitter@1.1.0': + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + + '@protobufjs/fetch@1.1.0': + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/inquire@1.1.0': + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.0': + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + '@radix-ui/number@1.1.0': resolution: {integrity: sha512-V3gRzhVNU1ldS5XhAPTom1fOIo4ccrjjJgmE+LI2h/WaFpHmx0MQApT+KZHnx8abG6Avtfcz4WoEciMnpFT3HQ==} @@ -2589,6 +2628,12 @@ packages: code-red@1.0.4: resolution: {integrity: sha512-7qJWqItLA8/VPVlKJlFXU+NBlo/qyfs39aJcuMT/2ere32ZqvF5OSxgdM5xOfJJ7O429gg2HM47y8v9P+9wrNw==} + codemirror-copilot@0.0.7: + resolution: {integrity: sha512-2nJlFXN8mEpzbAio/4vnxpZnItJodnvJ2iO4XvJKI4FrmB7cpZR1l9twn65m6RrCI8QF09ECeQ9baD4hn38axA==} + peerDependencies: + '@codemirror/state': ^6.2.0 + '@codemirror/view': ^6.7.2 + codemirror@6.0.1: resolution: {integrity: sha512-J8j+nZ+CdWmIeFIGXEFbFPtpiYacFMDR8GlHK3IyHQJMCaVRfGx9NT+Hxivv1ckLWPvNdZqndbr/7lVhrf/Svg==} @@ -3902,6 +3947,9 @@ packages: lodash@4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + long@5.2.3: + resolution: {integrity: sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==} + loose-envify@1.4.0: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true @@ -4346,6 +4394,10 @@ packages: prop-types@15.8.1: resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} + protobufjs@7.4.0: + resolution: {integrity: sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==} + engines: {node: '>=12.0.0'} + proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -6045,6 +6097,29 @@ snapshots: '@pkgr/core@0.1.1': {} + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.4': {} + + '@protobufjs/eventemitter@1.1.0': {} + + '@protobufjs/fetch@1.1.0': + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/inquire@1.1.0': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.0': {} + '@radix-ui/number@1.1.0': {} '@radix-ui/primitive@1.0.1': @@ -7501,6 +7576,11 @@ snapshots: estree-walker: 3.0.3 periscopic: 3.1.0 + codemirror-copilot@0.0.7(@codemirror/state@6.4.1)(@codemirror/view@6.33.0): + dependencies: + '@codemirror/state': 6.4.1 + '@codemirror/view': 6.33.0 + codemirror@6.0.1(@lezer/common@1.2.1): dependencies: '@codemirror/autocomplete': 6.18.1(@codemirror/language@6.10.2)(@codemirror/state@6.4.1)(@codemirror/view@6.33.0)(@lezer/common@1.2.1) @@ -8948,6 +9028,8 @@ snapshots: lodash@4.17.21: {} + long@5.2.3: {} + loose-envify@1.4.0: dependencies: js-tokens: 4.0.0 @@ -9369,6 +9451,21 @@ snapshots: object-assign: 4.1.1 react-is: 16.13.1 + protobufjs@7.4.0: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/node': 22.5.4 + long: 5.2.3 + proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 From 8b9eb03f2914e5e7c72a2fa01c26c9a551a3a121 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 8 Oct 2024 14:49:42 -0400 Subject: [PATCH 04/17] feat: use the user provided codeium api key if available when making requests --- packages/api/ai/autocomplete/index.mts | 11 ++++++++--- packages/api/server/http.mts | 4 +++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/api/ai/autocomplete/index.mts b/packages/api/ai/autocomplete/index.mts index fa0337a9..b580580d 100644 --- a/packages/api/ai/autocomplete/index.mts +++ b/packages/api/ai/autocomplete/index.mts @@ -8,7 +8,11 @@ import { type CodiumCompletionResult } from "@srcbook/shared"; const EDITOR_API_KEY = 'd49954eb-cfba-4992-980f-d8fb37f0e942'; const LANGUAGE_SERVER_PROTO_FILE_PATH = path.join(__dirname, "language_server.proto"); -export async function runCodiumAiAutocomplete(source: string, cursorOffset: number): Promise { +export async function runCodiumAiAutocomplete( + apiKey: string | null, + source: string, + cursorOffset: number, +): Promise { const protos = await protobuf.load(LANGUAGE_SERVER_PROTO_FILE_PATH) const GetCompletionsRequest = protos.lookupType("GetCompletionsRequest"); const Metadata = protos.lookupType("Metadata"); @@ -18,13 +22,14 @@ export async function runCodiumAiAutocomplete(source: string, cursorOffset: numb const GetCompletionsResponse = protos.lookupType("GetCompletionsResponse"); const sessionId = `react-editor-${crypto.randomUUID()}`; + const apiKey = apiKey ?? EDITOR_API_KEY; const payload = { otherDocuments: [], metadata: Metadata.create({ ideName: 'web', extensionVersion: '1.0.12', - apiKey: 'd49954eb-cfba-4992-980f-d8fb37f0e942', + apiKey, ideVersion: 'unknown', extensionName: '@codeium/react-code-editor', sessionId, @@ -56,7 +61,7 @@ export async function runCodiumAiAutocomplete(source: string, cursorOffset: numb headers: { 'Connect-Protocol-Version': '1', 'Content-Type': 'application/proto', - Authorization: `Basic ${EDITOR_API_KEY}-${sessionId}`, + Authorization: `Basic ${apiKey}-${sessionId}`, }, }); // console.log('RESPONSE:', response.status); diff --git a/packages/api/server/http.mts b/packages/api/server/http.mts index 51219b26..afffd45b 100644 --- a/packages/api/server/http.mts +++ b/packages/api/server/http.mts @@ -352,9 +352,11 @@ router.post('/feedback', cors(), async (req, res) => { router.options('/ai-autocomplete', cors()); router.post('/ai-autocomplete', cors(), async (req, res) => { const { source, cursorOffset } = req.body; + const config = await getConfig(); + let result; try { - result = await runCodiumAiAutocomplete(source, cursorOffset); + result = await runCodiumAiAutocomplete(config.codeiumApiKey, source, cursorOffset); } catch (err) { console.error('Error running ai autocomplete:', err); return res.json({ error: true }); From 2f43f5093f124a5be6ab4ea7295be40c389cf951 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 9 Oct 2024 13:14:53 -0400 Subject: [PATCH 05/17] fix: address issue in cell autocomplete due to an off by one in the cursor position --- packages/web/src/components/cells/code.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/web/src/components/cells/code.tsx b/packages/web/src/components/cells/code.tsx index 89aede9b..35f92be3 100644 --- a/packages/web/src/components/cells/code.tsx +++ b/packages/web/src/components/cells/code.tsx @@ -384,13 +384,12 @@ export default function ControlledCodeCell(props: Props) { inlineCopilot(async (prefix, suffix) => { let response; try { - response = await runCodiumAiAutocomplete(prefix+suffix, prefix.length-1); + response = await runCodiumAiAutocomplete(prefix+suffix, prefix.length); } catch (err) { console.error('Error fetching ai autocomplete suggestion:', err); return ""; } - console.log('AUTOCOMPLETE RESPONSE:', response); if (response.error) { return ""; } From 1a8a643f592790fc4b98e09b77a60e6b2ca51512 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 9 Oct 2024 14:03:58 -0400 Subject: [PATCH 06/17] feat: relocate codeium api request from server to client As part of this, I had to build the protobuf file into a set of "json definitions" that I could more easily include in a javascript bundle vs the raw bytes of a proto file. --- packages/api/package.json | 2 - packages/api/server/http.mts | 16 - packages/web/package.json | 2 + packages/web/src/components/cells/code.tsx | 17 +- .../lib/ai-autocomplete}/codeium_common.proto | 0 .../src/lib/ai-autocomplete/index.ts} | 24 +- .../ai-autocomplete/language-server-proto.ts | 522 ++++++++++++++++++ .../ai-autocomplete}/language_server.proto | 3 + packages/web/src/lib/server.ts | 14 - pnpm-lock.yaml | 12 +- 10 files changed, 554 insertions(+), 58 deletions(-) rename packages/{api/ai/autocomplete => web/src/lib/ai-autocomplete}/codeium_common.proto (100%) rename packages/{api/ai/autocomplete/index.mts => web/src/lib/ai-autocomplete/index.ts} (72%) create mode 100644 packages/web/src/lib/ai-autocomplete/language-server-proto.ts rename packages/{api/ai/autocomplete => web/src/lib/ai-autocomplete}/language_server.proto (98%) diff --git a/packages/api/package.json b/packages/api/package.json index 064aaddb..d89945b6 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -31,10 +31,8 @@ "depcheck": "^1.4.7", "drizzle-orm": "^0.33.0", "express": "^4.20.0", - "long": "^5.2.3", "marked": "catalog:", "posthog-node": "^4.2.0", - "protobufjs": "^7.4.0", "ws": "catalog:", "zod": "catalog:" }, diff --git a/packages/api/server/http.mts b/packages/api/server/http.mts index afffd45b..6a5096cc 100644 --- a/packages/api/server/http.mts +++ b/packages/api/server/http.mts @@ -14,7 +14,6 @@ import { exportSrcmdText, } from '../session.mjs'; import { generateCells, generateSrcbook, healthcheck } from '../ai/generate.mjs'; -import { runCodiumAiAutocomplete } from '../ai/autocomplete/index.mjs'; import { getConfig, updateConfig, @@ -349,21 +348,6 @@ router.post('/feedback', cors(), async (req, res) => { return res.json({ success: result.ok }); }); -router.options('/ai-autocomplete', cors()); -router.post('/ai-autocomplete', cors(), async (req, res) => { - const { source, cursorOffset } = req.body; - const config = await getConfig(); - - let result; - try { - result = await runCodiumAiAutocomplete(config.codeiumApiKey, source, cursorOffset); - } catch (err) { - console.error('Error running ai autocomplete:', err); - return res.json({ error: true }); - } - return res.json({ error: false, result }); -}); - type NpmSearchResult = { package: { name: string; diff --git a/packages/web/package.json b/packages/web/package.json index 56778dc2..cac1b19b 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -27,8 +27,10 @@ "clsx": "^2.1.1", "codemirror": "^6.0.1", "codemirror-copilot": "^0.0.7", + "long": "^5.2.3", "lucide-react": "^0.439.0", "marked": "catalog:", + "protobufjs": "^7.4.0", "react": "^18.3.1", "react-dom": "^18.3.1", "react-dropzone": "^14.2.3", diff --git a/packages/web/src/components/cells/code.tsx b/packages/web/src/components/cells/code.tsx index 35f92be3..86e2846e 100644 --- a/packages/web/src/components/cells/code.tsx +++ b/packages/web/src/components/cells/code.tsx @@ -18,7 +18,8 @@ import { SessionChannel } from '@/clients/websocket'; import { useCells } from '@srcbook/components/src/components/use-cell'; import { mapCMLocationToTsServer, mapTsServerLocationToCM } from './util'; import { toast } from 'sonner'; -import { getFileContent, runCodiumAiAutocomplete } from '@/lib/server'; +import { getFileContent } from '@/lib/server'; +import { runCodiumAiAutocomplete } from '@/lib/ai-autocomplete'; import { tsHover } from '@/components/cells/hover'; import { autocompletion } from '@codemirror/autocomplete'; import { type Diagnostic, linter } from '@codemirror/lint'; @@ -137,7 +138,7 @@ export default function ControlledCodeCell(props: Props) { const [prompt, setPrompt] = useState(''); const [newSource, setNewSource] = useState(''); const [fullscreen, setFullscreen] = useState(false); - const { aiEnabled } = useSettings(); + const { aiEnabled, codeiumApiKey } = useSettings(); const [isModalOpen, setIsModalOpen] = useState(false); const [modalContent, setModalContent] = useState(''); @@ -384,17 +385,17 @@ export default function ControlledCodeCell(props: Props) { inlineCopilot(async (prefix, suffix) => { let response; try { - response = await runCodiumAiAutocomplete(prefix+suffix, prefix.length); + response = await runCodiumAiAutocomplete( + codeiumApiKey ?? null, + prefix+suffix, + prefix.length, + ); } catch (err) { console.error('Error fetching ai autocomplete suggestion:', err); return ""; } - if (response.error) { - return ""; - } - - const completionItems = response.result.completionItems ?? []; + const completionItems = response.completionItems ?? []; const mostLikelyCompletionScore = Math.min(...completionItems.map(item => item.completion.score)); const mostLikelyCompletion = completionItems.find(item => item.completion.score === mostLikelyCompletionScore); diff --git a/packages/api/ai/autocomplete/codeium_common.proto b/packages/web/src/lib/ai-autocomplete/codeium_common.proto similarity index 100% rename from packages/api/ai/autocomplete/codeium_common.proto rename to packages/web/src/lib/ai-autocomplete/codeium_common.proto diff --git a/packages/api/ai/autocomplete/index.mts b/packages/web/src/lib/ai-autocomplete/index.ts similarity index 72% rename from packages/api/ai/autocomplete/index.mts rename to packages/web/src/lib/ai-autocomplete/index.ts index b580580d..acbe75f0 100644 --- a/packages/api/ai/autocomplete/index.mts +++ b/packages/web/src/lib/ai-autocomplete/index.ts @@ -1,28 +1,28 @@ -import path from 'node:path'; import protobuf from 'protobufjs'; import Long from 'long'; import { type CodiumCompletionResult } from "@srcbook/shared"; +import languageServerProto from "./language-server-proto"; + // NOTE: this EDITOR_API_KEY value was just included as a raw string in // @codeium/react-code-editor. This seems to not be a secret? const EDITOR_API_KEY = 'd49954eb-cfba-4992-980f-d8fb37f0e942'; -const LANGUAGE_SERVER_PROTO_FILE_PATH = path.join(__dirname, "language_server.proto"); export async function runCodiumAiAutocomplete( - apiKey: string | null, + optionalApiKey: string | null, source: string, cursorOffset: number, ): Promise { - const protos = await protobuf.load(LANGUAGE_SERVER_PROTO_FILE_PATH) - const GetCompletionsRequest = protos.lookupType("GetCompletionsRequest"); - const Metadata = protos.lookupType("Metadata"); - const DocumentInfo = protos.lookupType("Document"); - const EditorOptions = protos.lookupType("EditorOptions"); - const Language = protos.lookupEnum("Language"); - const GetCompletionsResponse = protos.lookupType("GetCompletionsResponse"); + const protos = protobuf.Root.fromJSON(languageServerProto as protobuf.INamespace); + const GetCompletionsRequest = protos.lookupType("exa.language_server_pb.GetCompletionsRequest"); + const Metadata = protos.lookupType("exa.codeium_common_pb.Metadata"); + const DocumentInfo = protos.lookupType("exa.language_server_pb.Document"); + const EditorOptions = protos.lookupType("exa.codeium_common_pb.EditorOptions"); + const Language = protos.lookupEnum("exa.codeium_common_pb.Language"); + const GetCompletionsResponse = protos.lookupType("exa.language_server_pb.GetCompletionsResponse"); const sessionId = `react-editor-${crypto.randomUUID()}`; - const apiKey = apiKey ?? EDITOR_API_KEY; + const apiKey = optionalApiKey ?? EDITOR_API_KEY; const payload = { otherDocuments: [], @@ -74,5 +74,5 @@ export async function runCodiumAiAutocomplete( // console.log(item.completion.text); // } - return responseBody; + return responseBody.toJSON(); } diff --git a/packages/web/src/lib/ai-autocomplete/language-server-proto.ts b/packages/web/src/lib/ai-autocomplete/language-server-proto.ts new file mode 100644 index 00000000..408ce80b --- /dev/null +++ b/packages/web/src/lib/ai-autocomplete/language-server-proto.ts @@ -0,0 +1,522 @@ +export default { + "options": { + "syntax": "proto3" + }, + "nested": { + "exa": { + "nested": { + "language_server_pb": { + "options": { + "go_package": "github.com/Exafunction/Exafunction/exa/language_server_pb" + }, + "nested": { + "LanguageServerService": { + "methods": { + "GetCompletions": { + "requestType": "GetCompletionsRequest", + "responseType": "GetCompletionsResponse" + }, + "AcceptCompletion": { + "requestType": "AcceptCompletionRequest", + "responseType": "AcceptCompletionResponse" + }, + "GetAuthToken": { + "requestType": "GetAuthTokenRequest", + "responseType": "GetAuthTokenResponse" + } + } + }, + "MultilineConfig": { + "fields": { + "threshold": { + "type": "float", + "id": 1 + } + } + }, + "GetCompletionsRequest": { + "fields": { + "metadata": { + "type": "codeium_common_pb.Metadata", + "id": 1 + }, + "document": { + "type": "Document", + "id": 2 + }, + "editorOptions": { + "type": "codeium_common_pb.EditorOptions", + "id": 3 + }, + "otherDocuments": { + "rule": "repeated", + "type": "Document", + "id": 5 + }, + "experimentConfig": { + "type": "ExperimentConfig", + "id": 7 + }, + "modelName": { + "type": "string", + "id": 10 + }, + "multilineConfig": { + "type": "MultilineConfig", + "id": 13 + } + } + }, + "GetCompletionsResponse": { + "fields": { + "state": { + "type": "State", + "id": 1 + }, + "completionItems": { + "rule": "repeated", + "type": "CompletionItem", + "id": 2 + } + } + }, + "AcceptCompletionRequest": { + "fields": { + "metadata": { + "type": "codeium_common_pb.Metadata", + "id": 1 + }, + "completionId": { + "type": "string", + "id": 2 + } + } + }, + "AcceptCompletionResponse": { + "fields": {} + }, + "GetAuthTokenRequest": { + "fields": {} + }, + "GetAuthTokenResponse": { + "fields": { + "authToken": { + "type": "string", + "id": 1 + }, + "uuid": { + "type": "string", + "id": 2 + } + } + }, + "DocumentPosition": { + "fields": { + "row": { + "type": "uint64", + "id": 1 + }, + "col": { + "type": "uint64", + "id": 2 + } + } + }, + "Document": { + "fields": { + "absolutePath": { + "type": "string", + "id": 1 + }, + "relativePath": { + "type": "string", + "id": 2 + }, + "text": { + "type": "string", + "id": 3 + }, + "editorLanguage": { + "type": "string", + "id": 4 + }, + "language": { + "type": "codeium_common_pb.Language", + "id": 5 + }, + "cursorOffset": { + "type": "uint64", + "id": 6 + }, + "cursorPosition": { + "type": "DocumentPosition", + "id": 8 + }, + "lineEnding": { + "type": "string", + "id": 7 + } + } + }, + "ExperimentConfig": { + "fields": { + "forceEnableExperiments": { + "rule": "repeated", + "type": "codeium_common_pb.ExperimentKey", + "id": 1 + } + } + }, + "CodeiumState": { + "values": { + "CODEIUM_STATE_UNSPECIFIED": 0, + "CODEIUM_STATE_INACTIVE": 1, + "CODEIUM_STATE_PROCESSING": 2, + "CODEIUM_STATE_SUCCESS": 3, + "CODEIUM_STATE_WARNING": 4, + "CODEIUM_STATE_ERROR": 5 + } + }, + "State": { + "fields": { + "state": { + "type": "CodeiumState", + "id": 1 + }, + "message": { + "type": "string", + "id": 2 + } + } + }, + "LineType": { + "values": { + "LINE_TYPE_UNSPECIFIED": 0, + "LINE_TYPE_SINGLE": 1, + "LINE_TYPE_MULTI": 2 + } + }, + "Range": { + "fields": { + "startOffset": { + "type": "uint64", + "id": 1 + }, + "endOffset": { + "type": "uint64", + "id": 2 + }, + "startPosition": { + "type": "DocumentPosition", + "id": 3 + }, + "endPosition": { + "type": "DocumentPosition", + "id": 4 + } + } + }, + "Suffix": { + "fields": { + "text": { + "type": "string", + "id": 1 + }, + "deltaCursorOffset": { + "type": "int64", + "id": 2 + } + } + }, + "CompletionPartType": { + "values": { + "COMPLETION_PART_TYPE_UNSPECIFIED": 0, + "COMPLETION_PART_TYPE_INLINE": 1, + "COMPLETION_PART_TYPE_BLOCK": 2, + "COMPLETION_PART_TYPE_INLINE_MASK": 3 + } + }, + "CompletionPart": { + "fields": { + "text": { + "type": "string", + "id": 1 + }, + "offset": { + "type": "uint64", + "id": 2 + }, + "type": { + "type": "CompletionPartType", + "id": 3 + }, + "prefix": { + "type": "string", + "id": 4 + }, + "line": { + "type": "uint64", + "id": 5 + } + } + }, + "CompletionItem": { + "fields": { + "completion": { + "type": "codeium_common_pb.Completion", + "id": 1 + }, + "suffix": { + "type": "Suffix", + "id": 5 + }, + "range": { + "type": "Range", + "id": 2 + }, + "source": { + "type": "codeium_common_pb.CompletionSource", + "id": 3 + }, + "completionParts": { + "rule": "repeated", + "type": "CompletionPart", + "id": 8 + } + } + } + } + }, + "codeium_common_pb": { + "options": { + "go_package": "github.com/Exafunction/Exafunction/exa/codeium_common_pb" + }, + "nested": { + "ExperimentKey": { + "values": { + "UNSPECIFIED": 0, + "JUPYTER_FORMAT": 77 + } + }, + "Completion": { + "fields": { + "completionId": { + "type": "string", + "id": 1 + }, + "text": { + "type": "string", + "id": 2 + }, + "prefix": { + "type": "string", + "id": 3 + }, + "stop": { + "type": "string", + "id": 4 + }, + "score": { + "type": "double", + "id": 5 + }, + "tokens": { + "rule": "repeated", + "type": "uint64", + "id": 6 + }, + "decodedTokens": { + "rule": "repeated", + "type": "string", + "id": 7 + }, + "probabilities": { + "rule": "repeated", + "type": "double", + "id": 8 + }, + "adjustedProbabilities": { + "rule": "repeated", + "type": "double", + "id": 9 + }, + "generatedLength": { + "type": "uint64", + "id": 10 + } + } + }, + "AuthSource": { + "values": { + "AUTH_SOURCE_CODEIUM": 0 + } + }, + "Metadata": { + "fields": { + "ideName": { + "type": "string", + "id": 1 + }, + "ideVersion": { + "type": "string", + "id": 7 + }, + "extensionName": { + "type": "string", + "id": 12 + }, + "extensionVersion": { + "type": "string", + "id": 2 + }, + "apiKey": { + "type": "string", + "id": 3 + }, + "locale": { + "type": "string", + "id": 4 + }, + "sessionId": { + "type": "string", + "id": 10 + }, + "requestId": { + "type": "uint64", + "id": 9 + }, + "userAgent": { + "type": "string", + "id": 13 + }, + "url": { + "type": "string", + "id": 14 + }, + "authSource": { + "type": "AuthSource", + "id": 15 + } + } + }, + "EditorOptions": { + "fields": { + "tabSize": { + "type": "uint64", + "id": 1 + }, + "insertSpaces": { + "type": "bool", + "id": 2 + } + } + }, + "Event": { + "fields": { + "eventType": { + "type": "EventType", + "id": 1 + }, + "eventJson": { + "type": "string", + "id": 2 + }, + "timestampUnixMs": { + "type": "int64", + "id": 3 + } + } + }, + "EventType": { + "values": { + "EVENT_TYPE_UNSPECIFIED": 0, + "EVENT_TYPE_ENABLE_CODEIUM": 1, + "EVENT_TYPE_DISABLE_CODEIUM": 2, + "EVENT_TYPE_SHOW_PREVIOUS_COMPLETION": 3, + "EVENT_TYPE_SHOW_NEXT_COMPLETION": 4 + } + }, + "CompletionSource": { + "values": { + "COMPLETION_SOURCE_UNSPECIFIED": 0, + "COMPLETION_SOURCE_TYPING_AS_SUGGESTED": 1, + "COMPLETION_SOURCE_CACHE": 2, + "COMPLETION_SOURCE_NETWORK": 3 + } + }, + "Language": { + "values": { + "LANGUAGE_UNSPECIFIED": 0, + "LANGUAGE_C": 1, + "LANGUAGE_CLOJURE": 2, + "LANGUAGE_COFFEESCRIPT": 3, + "LANGUAGE_CPP": 4, + "LANGUAGE_CSHARP": 5, + "LANGUAGE_CSS": 6, + "LANGUAGE_CUDACPP": 7, + "LANGUAGE_DOCKERFILE": 8, + "LANGUAGE_GO": 9, + "LANGUAGE_GROOVY": 10, + "LANGUAGE_HANDLEBARS": 11, + "LANGUAGE_HASKELL": 12, + "LANGUAGE_HCL": 13, + "LANGUAGE_HTML": 14, + "LANGUAGE_INI": 15, + "LANGUAGE_JAVA": 16, + "LANGUAGE_JAVASCRIPT": 17, + "LANGUAGE_JSON": 18, + "LANGUAGE_JULIA": 19, + "LANGUAGE_KOTLIN": 20, + "LANGUAGE_LATEX": 21, + "LANGUAGE_LESS": 22, + "LANGUAGE_LUA": 23, + "LANGUAGE_MAKEFILE": 24, + "LANGUAGE_MARKDOWN": 25, + "LANGUAGE_OBJECTIVEC": 26, + "LANGUAGE_OBJECTIVECPP": 27, + "LANGUAGE_PERL": 28, + "LANGUAGE_PHP": 29, + "LANGUAGE_PLAINTEXT": 30, + "LANGUAGE_PROTOBUF": 31, + "LANGUAGE_PBTXT": 32, + "LANGUAGE_PYTHON": 33, + "LANGUAGE_R": 34, + "LANGUAGE_RUBY": 35, + "LANGUAGE_RUST": 36, + "LANGUAGE_SASS": 37, + "LANGUAGE_SCALA": 38, + "LANGUAGE_SCSS": 39, + "LANGUAGE_SHELL": 40, + "LANGUAGE_SQL": 41, + "LANGUAGE_STARLARK": 42, + "LANGUAGE_SWIFT": 43, + "LANGUAGE_TSX": 44, + "LANGUAGE_TYPESCRIPT": 45, + "LANGUAGE_VISUALBASIC": 46, + "LANGUAGE_VUE": 47, + "LANGUAGE_XML": 48, + "LANGUAGE_XSL": 49, + "LANGUAGE_YAML": 50, + "LANGUAGE_SVELTE": 51, + "LANGUAGE_TOML": 52, + "LANGUAGE_DART": 53, + "LANGUAGE_RST": 54, + "LANGUAGE_OCAML": 55, + "LANGUAGE_CMAKE": 56, + "LANGUAGE_PASCAL": 57, + "LANGUAGE_ELIXIR": 58, + "LANGUAGE_FSHARP": 59, + "LANGUAGE_LISP": 60, + "LANGUAGE_MATLAB": 61, + "LANGUAGE_POWERSHELL": 62, + "LANGUAGE_SOLIDITY": 63, + "LANGUAGE_ADA": 64, + "LANGUAGE_OCAML_INTERFACE": 65 + } + } + } + } + } + } + } +} diff --git a/packages/api/ai/autocomplete/language_server.proto b/packages/web/src/lib/ai-autocomplete/language_server.proto similarity index 98% rename from packages/api/ai/autocomplete/language_server.proto rename to packages/web/src/lib/ai-autocomplete/language_server.proto index 84fcb518..278fce99 100644 --- a/packages/api/ai/autocomplete/language_server.proto +++ b/packages/web/src/lib/ai-autocomplete/language_server.proto @@ -1,3 +1,6 @@ +// NOTE: to generate a new set of json definitions, run: +// npm run generate-codeium-proto-json + // Copyright Exafunction, Inc. syntax = "proto3"; diff --git a/packages/web/src/lib/server.ts b/packages/web/src/lib/server.ts index fb3d7043..f72d4989 100644 --- a/packages/web/src/lib/server.ts +++ b/packages/web/src/lib/server.ts @@ -24,20 +24,6 @@ export async function getFileContent(filename: string) { return await file_response.json(); } -export async function runCodiumAiAutocomplete(source: string, cursorOffset: number): Promise<{error: false, result: CodiumCompletionResult} | { error: true }> { - const file_response = await fetch(API_BASE_URL + '/ai-autocomplete', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - source, - cursorOffset, - }), - }); - return await file_response.json(); -} - interface CreateSrcbookRequestType { path: string; name: string; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d647e2cd..2ebcbce1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -78,18 +78,12 @@ importers: express: specifier: ^4.20.0 version: 4.20.0 - long: - specifier: ^5.2.3 - version: 5.2.3 marked: specifier: 'catalog:' version: 14.1.2 posthog-node: specifier: ^4.2.0 version: 4.2.0 - protobufjs: - specifier: ^7.4.0 - version: 7.4.0 ws: specifier: 'catalog:' version: 8.18.0 @@ -328,12 +322,18 @@ importers: codemirror-copilot: specifier: ^0.0.7 version: 0.0.7(@codemirror/state@6.4.1)(@codemirror/view@6.33.0) + long: + specifier: ^5.2.3 + version: 5.2.3 lucide-react: specifier: ^0.439.0 version: 0.439.0(react@18.3.1) marked: specifier: 'catalog:' version: 14.1.2 + protobufjs: + specifier: ^7.4.0 + version: 7.4.0 react: specifier: ^18.3.1 version: 18.3.1 From 4f8b323f7ab89952ebdadee8b830cd5a3dc6f906 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 9 Oct 2024 14:09:16 -0400 Subject: [PATCH 07/17] feat: add subcommmand to rebuild codeium proto json definitions --- packages/web/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/web/package.json b/packages/web/package.json index cac1b19b..f41bfa9f 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -10,7 +10,8 @@ "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", "format": "prettier --write .", "preview": "vite preview", - "check-types": "tsc" + "check-types": "tsc", + "generate-codeium-proto-json": "node --eval 'console.log(\"export default\", JSON.stringify(require(\"protobufjs\").loadSync(\"src/lib/ai-autocomplete/language_server.proto\").toJSON(), null, 2));' > src/lib/ai-autocomplete/languageServerProto.ts" }, "dependencies": { "@codemirror/autocomplete": "^6.18.1", From c6d5bb179db6730c42d54eea70d62901780b890d Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 9 Oct 2024 14:11:44 -0400 Subject: [PATCH 08/17] fix: run npm run format --- packages/shared/src/types/ai-autocomplete.mts | 27 +- packages/web/src/components/cells/code.tsx | 18 +- packages/web/src/lib/ai-autocomplete/index.ts | 39 +- .../ai-autocomplete/language-server-proto.ts | 982 +++++++++--------- .../src/routes/settings-codeium-callback.tsx | 52 +- packages/web/src/routes/settings.tsx | 39 +- 6 files changed, 584 insertions(+), 573 deletions(-) diff --git a/packages/shared/src/types/ai-autocomplete.mts b/packages/shared/src/types/ai-autocomplete.mts index 8ff2bd38..b62fc6ad 100644 --- a/packages/shared/src/types/ai-autocomplete.mts +++ b/packages/shared/src/types/ai-autocomplete.mts @@ -15,15 +15,14 @@ export type CodiumCompletionItem = { text: string; offset: string; prefix: string; - type: ( - | "COMPLETION_PART_TYPE_UNSPECIFIED" + type: + | 'COMPLETION_PART_TYPE_UNSPECIFIED' // Single-line completion parts that appear within an existing line of text. - | "COMPLETION_PART_TYPE_INLINE" + | 'COMPLETION_PART_TYPE_INLINE' // Possibly multi-line completion parts that appear below an existing line of text. - | "COMPLETION_PART_TYPE_BLOCK" + | 'COMPLETION_PART_TYPE_BLOCK' // Like COMPLETION_PART_TYPE_INLINE, but overwrites the existing text. - | "COMPLETION_PART_TYPE_INLINE_MASK" - ); + | 'COMPLETION_PART_TYPE_INLINE_MASK'; }>; range: { endOffset: string; @@ -35,15 +34,13 @@ export type CodiumCompletionItem = { export type CodiumCompletionResult = { completionItems?: Array; state: { - state: ( - | "CODEIUM_STATE_UNSPECIFIED" - | "CODEIUM_STATE_INACTIVE" - | "CODEIUM_STATE_PROCESSING" - | "CODEIUM_STATE_SUCCESS" - | "CODEIUM_STATE_WARNING" - | "CODEIUM_STATE_ERROR" - ), + state: + | 'CODEIUM_STATE_UNSPECIFIED' + | 'CODEIUM_STATE_INACTIVE' + | 'CODEIUM_STATE_PROCESSING' + | 'CODEIUM_STATE_SUCCESS' + | 'CODEIUM_STATE_WARNING' + | 'CODEIUM_STATE_ERROR'; status: string; }; }; - diff --git a/packages/web/src/components/cells/code.tsx b/packages/web/src/components/cells/code.tsx index 86e2846e..0d322671 100644 --- a/packages/web/src/components/cells/code.tsx +++ b/packages/web/src/components/cells/code.tsx @@ -35,7 +35,7 @@ import CodeMirror, { } from '@uiw/react-codemirror'; import useTheme from '@srcbook/components/src/components/use-theme'; import { Dialog, DialogContent } from '@srcbook/components/src/components/ui/dialog'; -import { inlineCopilot } from "codemirror-copilot"; +import { inlineCopilot } from 'codemirror-copilot'; function tsLinter( cell: CodeCellType, @@ -387,19 +387,23 @@ export default function ControlledCodeCell(props: Props) { try { response = await runCodiumAiAutocomplete( codeiumApiKey ?? null, - prefix+suffix, + prefix + suffix, prefix.length, ); } catch (err) { console.error('Error fetching ai autocomplete suggestion:', err); - return ""; + return ''; } const completionItems = response.completionItems ?? []; - const mostLikelyCompletionScore = Math.min(...completionItems.map(item => item.completion.score)); - const mostLikelyCompletion = completionItems.find(item => item.completion.score === mostLikelyCompletionScore); - - return mostLikelyCompletion?.completionParts[0]?.text ?? ""; + const mostLikelyCompletionScore = Math.min( + ...completionItems.map((item) => item.completion.score), + ); + const mostLikelyCompletion = completionItems.find( + (item) => item.completion.score === mostLikelyCompletionScore, + ); + + return mostLikelyCompletion?.completionParts[0]?.text ?? ''; }, DEBOUNCE_DELAY), ); extensions.push( diff --git a/packages/web/src/lib/ai-autocomplete/index.ts b/packages/web/src/lib/ai-autocomplete/index.ts index acbe75f0..f0cba894 100644 --- a/packages/web/src/lib/ai-autocomplete/index.ts +++ b/packages/web/src/lib/ai-autocomplete/index.ts @@ -1,8 +1,8 @@ import protobuf from 'protobufjs'; import Long from 'long'; -import { type CodiumCompletionResult } from "@srcbook/shared"; +import { type CodiumCompletionResult } from '@srcbook/shared'; -import languageServerProto from "./language-server-proto"; +import languageServerProto from './language-server-proto'; // NOTE: this EDITOR_API_KEY value was just included as a raw string in // @codeium/react-code-editor. This seems to not be a secret? @@ -14,12 +14,12 @@ export async function runCodiumAiAutocomplete( cursorOffset: number, ): Promise { const protos = protobuf.Root.fromJSON(languageServerProto as protobuf.INamespace); - const GetCompletionsRequest = protos.lookupType("exa.language_server_pb.GetCompletionsRequest"); - const Metadata = protos.lookupType("exa.codeium_common_pb.Metadata"); - const DocumentInfo = protos.lookupType("exa.language_server_pb.Document"); - const EditorOptions = protos.lookupType("exa.codeium_common_pb.EditorOptions"); - const Language = protos.lookupEnum("exa.codeium_common_pb.Language"); - const GetCompletionsResponse = protos.lookupType("exa.language_server_pb.GetCompletionsResponse"); + const GetCompletionsRequest = protos.lookupType('exa.language_server_pb.GetCompletionsRequest'); + const Metadata = protos.lookupType('exa.codeium_common_pb.Metadata'); + const DocumentInfo = protos.lookupType('exa.language_server_pb.Document'); + const EditorOptions = protos.lookupType('exa.codeium_common_pb.EditorOptions'); + const Language = protos.lookupEnum('exa.codeium_common_pb.Language'); + const GetCompletionsResponse = protos.lookupType('exa.language_server_pb.GetCompletionsResponse'); const sessionId = `react-editor-${crypto.randomUUID()}`; const apiKey = optionalApiKey ?? EDITOR_API_KEY; @@ -37,13 +37,13 @@ export async function runCodiumAiAutocomplete( document: DocumentInfo.create({ text: source, editorLanguage: 'javascript', - language: Language.getOption("JAVASCRIPT"), + language: Language.getOption('JAVASCRIPT'), cursorOffset: Long.fromValue(cursorOffset), lineEnding: '\n', }), editorOptions: EditorOptions.create({ tabSize: Long.fromValue(4), - insertSpaces: true + insertSpaces: true, }), }; @@ -55,15 +55,18 @@ export async function runCodiumAiAutocomplete( const requestData = GetCompletionsRequest.create(payload); const buffer = GetCompletionsRequest.encode(requestData).finish(); - const response = await fetch('https://web-backend.codeium.com/exa.language_server_pb.LanguageServerService/GetCompletions', { - method: 'POST', - body: buffer, - headers: { - 'Connect-Protocol-Version': '1', - 'Content-Type': 'application/proto', - Authorization: `Basic ${apiKey}-${sessionId}`, + const response = await fetch( + 'https://web-backend.codeium.com/exa.language_server_pb.LanguageServerService/GetCompletions', + { + method: 'POST', + body: buffer, + headers: { + 'Connect-Protocol-Version': '1', + 'Content-Type': 'application/proto', + Authorization: `Basic ${apiKey}-${sessionId}`, + }, }, - }); + ); // console.log('RESPONSE:', response.status); const responseBodyBytes = new Uint8Array(await response.arrayBuffer()); diff --git a/packages/web/src/lib/ai-autocomplete/language-server-proto.ts b/packages/web/src/lib/ai-autocomplete/language-server-proto.ts index 408ce80b..a5029361 100644 --- a/packages/web/src/lib/ai-autocomplete/language-server-proto.ts +++ b/packages/web/src/lib/ai-autocomplete/language-server-proto.ts @@ -1,522 +1,522 @@ export default { - "options": { - "syntax": "proto3" + options: { + syntax: 'proto3', }, - "nested": { - "exa": { - "nested": { - "language_server_pb": { - "options": { - "go_package": "github.com/Exafunction/Exafunction/exa/language_server_pb" + nested: { + exa: { + nested: { + language_server_pb: { + options: { + go_package: 'github.com/Exafunction/Exafunction/exa/language_server_pb', }, - "nested": { - "LanguageServerService": { - "methods": { - "GetCompletions": { - "requestType": "GetCompletionsRequest", - "responseType": "GetCompletionsResponse" - }, - "AcceptCompletion": { - "requestType": "AcceptCompletionRequest", - "responseType": "AcceptCompletionResponse" - }, - "GetAuthToken": { - "requestType": "GetAuthTokenRequest", - "responseType": "GetAuthTokenResponse" - } - } + nested: { + LanguageServerService: { + methods: { + GetCompletions: { + requestType: 'GetCompletionsRequest', + responseType: 'GetCompletionsResponse', + }, + AcceptCompletion: { + requestType: 'AcceptCompletionRequest', + responseType: 'AcceptCompletionResponse', + }, + GetAuthToken: { + requestType: 'GetAuthTokenRequest', + responseType: 'GetAuthTokenResponse', + }, + }, }, - "MultilineConfig": { - "fields": { - "threshold": { - "type": "float", - "id": 1 - } - } + MultilineConfig: { + fields: { + threshold: { + type: 'float', + id: 1, + }, + }, }, - "GetCompletionsRequest": { - "fields": { - "metadata": { - "type": "codeium_common_pb.Metadata", - "id": 1 - }, - "document": { - "type": "Document", - "id": 2 - }, - "editorOptions": { - "type": "codeium_common_pb.EditorOptions", - "id": 3 - }, - "otherDocuments": { - "rule": "repeated", - "type": "Document", - "id": 5 - }, - "experimentConfig": { - "type": "ExperimentConfig", - "id": 7 - }, - "modelName": { - "type": "string", - "id": 10 - }, - "multilineConfig": { - "type": "MultilineConfig", - "id": 13 - } - } + GetCompletionsRequest: { + fields: { + metadata: { + type: 'codeium_common_pb.Metadata', + id: 1, + }, + document: { + type: 'Document', + id: 2, + }, + editorOptions: { + type: 'codeium_common_pb.EditorOptions', + id: 3, + }, + otherDocuments: { + rule: 'repeated', + type: 'Document', + id: 5, + }, + experimentConfig: { + type: 'ExperimentConfig', + id: 7, + }, + modelName: { + type: 'string', + id: 10, + }, + multilineConfig: { + type: 'MultilineConfig', + id: 13, + }, + }, }, - "GetCompletionsResponse": { - "fields": { - "state": { - "type": "State", - "id": 1 - }, - "completionItems": { - "rule": "repeated", - "type": "CompletionItem", - "id": 2 - } - } + GetCompletionsResponse: { + fields: { + state: { + type: 'State', + id: 1, + }, + completionItems: { + rule: 'repeated', + type: 'CompletionItem', + id: 2, + }, + }, }, - "AcceptCompletionRequest": { - "fields": { - "metadata": { - "type": "codeium_common_pb.Metadata", - "id": 1 - }, - "completionId": { - "type": "string", - "id": 2 - } - } + AcceptCompletionRequest: { + fields: { + metadata: { + type: 'codeium_common_pb.Metadata', + id: 1, + }, + completionId: { + type: 'string', + id: 2, + }, + }, }, - "AcceptCompletionResponse": { - "fields": {} + AcceptCompletionResponse: { + fields: {}, }, - "GetAuthTokenRequest": { - "fields": {} + GetAuthTokenRequest: { + fields: {}, }, - "GetAuthTokenResponse": { - "fields": { - "authToken": { - "type": "string", - "id": 1 - }, - "uuid": { - "type": "string", - "id": 2 - } - } + GetAuthTokenResponse: { + fields: { + authToken: { + type: 'string', + id: 1, + }, + uuid: { + type: 'string', + id: 2, + }, + }, }, - "DocumentPosition": { - "fields": { - "row": { - "type": "uint64", - "id": 1 - }, - "col": { - "type": "uint64", - "id": 2 - } - } + DocumentPosition: { + fields: { + row: { + type: 'uint64', + id: 1, + }, + col: { + type: 'uint64', + id: 2, + }, + }, }, - "Document": { - "fields": { - "absolutePath": { - "type": "string", - "id": 1 - }, - "relativePath": { - "type": "string", - "id": 2 - }, - "text": { - "type": "string", - "id": 3 - }, - "editorLanguage": { - "type": "string", - "id": 4 - }, - "language": { - "type": "codeium_common_pb.Language", - "id": 5 - }, - "cursorOffset": { - "type": "uint64", - "id": 6 - }, - "cursorPosition": { - "type": "DocumentPosition", - "id": 8 - }, - "lineEnding": { - "type": "string", - "id": 7 - } - } + Document: { + fields: { + absolutePath: { + type: 'string', + id: 1, + }, + relativePath: { + type: 'string', + id: 2, + }, + text: { + type: 'string', + id: 3, + }, + editorLanguage: { + type: 'string', + id: 4, + }, + language: { + type: 'codeium_common_pb.Language', + id: 5, + }, + cursorOffset: { + type: 'uint64', + id: 6, + }, + cursorPosition: { + type: 'DocumentPosition', + id: 8, + }, + lineEnding: { + type: 'string', + id: 7, + }, + }, + }, + ExperimentConfig: { + fields: { + forceEnableExperiments: { + rule: 'repeated', + type: 'codeium_common_pb.ExperimentKey', + id: 1, + }, + }, }, - "ExperimentConfig": { - "fields": { - "forceEnableExperiments": { - "rule": "repeated", - "type": "codeium_common_pb.ExperimentKey", - "id": 1 - } - } + CodeiumState: { + values: { + CODEIUM_STATE_UNSPECIFIED: 0, + CODEIUM_STATE_INACTIVE: 1, + CODEIUM_STATE_PROCESSING: 2, + CODEIUM_STATE_SUCCESS: 3, + CODEIUM_STATE_WARNING: 4, + CODEIUM_STATE_ERROR: 5, + }, }, - "CodeiumState": { - "values": { - "CODEIUM_STATE_UNSPECIFIED": 0, - "CODEIUM_STATE_INACTIVE": 1, - "CODEIUM_STATE_PROCESSING": 2, - "CODEIUM_STATE_SUCCESS": 3, - "CODEIUM_STATE_WARNING": 4, - "CODEIUM_STATE_ERROR": 5 - } + State: { + fields: { + state: { + type: 'CodeiumState', + id: 1, + }, + message: { + type: 'string', + id: 2, + }, + }, }, - "State": { - "fields": { - "state": { - "type": "CodeiumState", - "id": 1 - }, - "message": { - "type": "string", - "id": 2 - } - } + LineType: { + values: { + LINE_TYPE_UNSPECIFIED: 0, + LINE_TYPE_SINGLE: 1, + LINE_TYPE_MULTI: 2, + }, }, - "LineType": { - "values": { - "LINE_TYPE_UNSPECIFIED": 0, - "LINE_TYPE_SINGLE": 1, - "LINE_TYPE_MULTI": 2 - } + Range: { + fields: { + startOffset: { + type: 'uint64', + id: 1, + }, + endOffset: { + type: 'uint64', + id: 2, + }, + startPosition: { + type: 'DocumentPosition', + id: 3, + }, + endPosition: { + type: 'DocumentPosition', + id: 4, + }, + }, }, - "Range": { - "fields": { - "startOffset": { - "type": "uint64", - "id": 1 - }, - "endOffset": { - "type": "uint64", - "id": 2 - }, - "startPosition": { - "type": "DocumentPosition", - "id": 3 - }, - "endPosition": { - "type": "DocumentPosition", - "id": 4 - } - } + Suffix: { + fields: { + text: { + type: 'string', + id: 1, + }, + deltaCursorOffset: { + type: 'int64', + id: 2, + }, + }, }, - "Suffix": { - "fields": { - "text": { - "type": "string", - "id": 1 - }, - "deltaCursorOffset": { - "type": "int64", - "id": 2 - } - } + CompletionPartType: { + values: { + COMPLETION_PART_TYPE_UNSPECIFIED: 0, + COMPLETION_PART_TYPE_INLINE: 1, + COMPLETION_PART_TYPE_BLOCK: 2, + COMPLETION_PART_TYPE_INLINE_MASK: 3, + }, }, - "CompletionPartType": { - "values": { - "COMPLETION_PART_TYPE_UNSPECIFIED": 0, - "COMPLETION_PART_TYPE_INLINE": 1, - "COMPLETION_PART_TYPE_BLOCK": 2, - "COMPLETION_PART_TYPE_INLINE_MASK": 3 - } + CompletionPart: { + fields: { + text: { + type: 'string', + id: 1, + }, + offset: { + type: 'uint64', + id: 2, + }, + type: { + type: 'CompletionPartType', + id: 3, + }, + prefix: { + type: 'string', + id: 4, + }, + line: { + type: 'uint64', + id: 5, + }, + }, }, - "CompletionPart": { - "fields": { - "text": { - "type": "string", - "id": 1 - }, - "offset": { - "type": "uint64", - "id": 2 - }, - "type": { - "type": "CompletionPartType", - "id": 3 - }, - "prefix": { - "type": "string", - "id": 4 - }, - "line": { - "type": "uint64", - "id": 5 - } - } + CompletionItem: { + fields: { + completion: { + type: 'codeium_common_pb.Completion', + id: 1, + }, + suffix: { + type: 'Suffix', + id: 5, + }, + range: { + type: 'Range', + id: 2, + }, + source: { + type: 'codeium_common_pb.CompletionSource', + id: 3, + }, + completionParts: { + rule: 'repeated', + type: 'CompletionPart', + id: 8, + }, + }, }, - "CompletionItem": { - "fields": { - "completion": { - "type": "codeium_common_pb.Completion", - "id": 1 - }, - "suffix": { - "type": "Suffix", - "id": 5 - }, - "range": { - "type": "Range", - "id": 2 - }, - "source": { - "type": "codeium_common_pb.CompletionSource", - "id": 3 - }, - "completionParts": { - "rule": "repeated", - "type": "CompletionPart", - "id": 8 - } - } - } - } + }, }, - "codeium_common_pb": { - "options": { - "go_package": "github.com/Exafunction/Exafunction/exa/codeium_common_pb" + codeium_common_pb: { + options: { + go_package: 'github.com/Exafunction/Exafunction/exa/codeium_common_pb', }, - "nested": { - "ExperimentKey": { - "values": { - "UNSPECIFIED": 0, - "JUPYTER_FORMAT": 77 - } + nested: { + ExperimentKey: { + values: { + UNSPECIFIED: 0, + JUPYTER_FORMAT: 77, + }, }, - "Completion": { - "fields": { - "completionId": { - "type": "string", - "id": 1 - }, - "text": { - "type": "string", - "id": 2 - }, - "prefix": { - "type": "string", - "id": 3 - }, - "stop": { - "type": "string", - "id": 4 - }, - "score": { - "type": "double", - "id": 5 - }, - "tokens": { - "rule": "repeated", - "type": "uint64", - "id": 6 - }, - "decodedTokens": { - "rule": "repeated", - "type": "string", - "id": 7 - }, - "probabilities": { - "rule": "repeated", - "type": "double", - "id": 8 - }, - "adjustedProbabilities": { - "rule": "repeated", - "type": "double", - "id": 9 - }, - "generatedLength": { - "type": "uint64", - "id": 10 - } - } + Completion: { + fields: { + completionId: { + type: 'string', + id: 1, + }, + text: { + type: 'string', + id: 2, + }, + prefix: { + type: 'string', + id: 3, + }, + stop: { + type: 'string', + id: 4, + }, + score: { + type: 'double', + id: 5, + }, + tokens: { + rule: 'repeated', + type: 'uint64', + id: 6, + }, + decodedTokens: { + rule: 'repeated', + type: 'string', + id: 7, + }, + probabilities: { + rule: 'repeated', + type: 'double', + id: 8, + }, + adjustedProbabilities: { + rule: 'repeated', + type: 'double', + id: 9, + }, + generatedLength: { + type: 'uint64', + id: 10, + }, + }, }, - "AuthSource": { - "values": { - "AUTH_SOURCE_CODEIUM": 0 - } + AuthSource: { + values: { + AUTH_SOURCE_CODEIUM: 0, + }, }, - "Metadata": { - "fields": { - "ideName": { - "type": "string", - "id": 1 - }, - "ideVersion": { - "type": "string", - "id": 7 - }, - "extensionName": { - "type": "string", - "id": 12 - }, - "extensionVersion": { - "type": "string", - "id": 2 - }, - "apiKey": { - "type": "string", - "id": 3 - }, - "locale": { - "type": "string", - "id": 4 - }, - "sessionId": { - "type": "string", - "id": 10 - }, - "requestId": { - "type": "uint64", - "id": 9 - }, - "userAgent": { - "type": "string", - "id": 13 - }, - "url": { - "type": "string", - "id": 14 - }, - "authSource": { - "type": "AuthSource", - "id": 15 - } - } + Metadata: { + fields: { + ideName: { + type: 'string', + id: 1, + }, + ideVersion: { + type: 'string', + id: 7, + }, + extensionName: { + type: 'string', + id: 12, + }, + extensionVersion: { + type: 'string', + id: 2, + }, + apiKey: { + type: 'string', + id: 3, + }, + locale: { + type: 'string', + id: 4, + }, + sessionId: { + type: 'string', + id: 10, + }, + requestId: { + type: 'uint64', + id: 9, + }, + userAgent: { + type: 'string', + id: 13, + }, + url: { + type: 'string', + id: 14, + }, + authSource: { + type: 'AuthSource', + id: 15, + }, + }, + }, + EditorOptions: { + fields: { + tabSize: { + type: 'uint64', + id: 1, + }, + insertSpaces: { + type: 'bool', + id: 2, + }, + }, }, - "EditorOptions": { - "fields": { - "tabSize": { - "type": "uint64", - "id": 1 - }, - "insertSpaces": { - "type": "bool", - "id": 2 - } - } + Event: { + fields: { + eventType: { + type: 'EventType', + id: 1, + }, + eventJson: { + type: 'string', + id: 2, + }, + timestampUnixMs: { + type: 'int64', + id: 3, + }, + }, }, - "Event": { - "fields": { - "eventType": { - "type": "EventType", - "id": 1 - }, - "eventJson": { - "type": "string", - "id": 2 - }, - "timestampUnixMs": { - "type": "int64", - "id": 3 - } - } + EventType: { + values: { + EVENT_TYPE_UNSPECIFIED: 0, + EVENT_TYPE_ENABLE_CODEIUM: 1, + EVENT_TYPE_DISABLE_CODEIUM: 2, + EVENT_TYPE_SHOW_PREVIOUS_COMPLETION: 3, + EVENT_TYPE_SHOW_NEXT_COMPLETION: 4, + }, }, - "EventType": { - "values": { - "EVENT_TYPE_UNSPECIFIED": 0, - "EVENT_TYPE_ENABLE_CODEIUM": 1, - "EVENT_TYPE_DISABLE_CODEIUM": 2, - "EVENT_TYPE_SHOW_PREVIOUS_COMPLETION": 3, - "EVENT_TYPE_SHOW_NEXT_COMPLETION": 4 - } + CompletionSource: { + values: { + COMPLETION_SOURCE_UNSPECIFIED: 0, + COMPLETION_SOURCE_TYPING_AS_SUGGESTED: 1, + COMPLETION_SOURCE_CACHE: 2, + COMPLETION_SOURCE_NETWORK: 3, + }, }, - "CompletionSource": { - "values": { - "COMPLETION_SOURCE_UNSPECIFIED": 0, - "COMPLETION_SOURCE_TYPING_AS_SUGGESTED": 1, - "COMPLETION_SOURCE_CACHE": 2, - "COMPLETION_SOURCE_NETWORK": 3 - } + Language: { + values: { + LANGUAGE_UNSPECIFIED: 0, + LANGUAGE_C: 1, + LANGUAGE_CLOJURE: 2, + LANGUAGE_COFFEESCRIPT: 3, + LANGUAGE_CPP: 4, + LANGUAGE_CSHARP: 5, + LANGUAGE_CSS: 6, + LANGUAGE_CUDACPP: 7, + LANGUAGE_DOCKERFILE: 8, + LANGUAGE_GO: 9, + LANGUAGE_GROOVY: 10, + LANGUAGE_HANDLEBARS: 11, + LANGUAGE_HASKELL: 12, + LANGUAGE_HCL: 13, + LANGUAGE_HTML: 14, + LANGUAGE_INI: 15, + LANGUAGE_JAVA: 16, + LANGUAGE_JAVASCRIPT: 17, + LANGUAGE_JSON: 18, + LANGUAGE_JULIA: 19, + LANGUAGE_KOTLIN: 20, + LANGUAGE_LATEX: 21, + LANGUAGE_LESS: 22, + LANGUAGE_LUA: 23, + LANGUAGE_MAKEFILE: 24, + LANGUAGE_MARKDOWN: 25, + LANGUAGE_OBJECTIVEC: 26, + LANGUAGE_OBJECTIVECPP: 27, + LANGUAGE_PERL: 28, + LANGUAGE_PHP: 29, + LANGUAGE_PLAINTEXT: 30, + LANGUAGE_PROTOBUF: 31, + LANGUAGE_PBTXT: 32, + LANGUAGE_PYTHON: 33, + LANGUAGE_R: 34, + LANGUAGE_RUBY: 35, + LANGUAGE_RUST: 36, + LANGUAGE_SASS: 37, + LANGUAGE_SCALA: 38, + LANGUAGE_SCSS: 39, + LANGUAGE_SHELL: 40, + LANGUAGE_SQL: 41, + LANGUAGE_STARLARK: 42, + LANGUAGE_SWIFT: 43, + LANGUAGE_TSX: 44, + LANGUAGE_TYPESCRIPT: 45, + LANGUAGE_VISUALBASIC: 46, + LANGUAGE_VUE: 47, + LANGUAGE_XML: 48, + LANGUAGE_XSL: 49, + LANGUAGE_YAML: 50, + LANGUAGE_SVELTE: 51, + LANGUAGE_TOML: 52, + LANGUAGE_DART: 53, + LANGUAGE_RST: 54, + LANGUAGE_OCAML: 55, + LANGUAGE_CMAKE: 56, + LANGUAGE_PASCAL: 57, + LANGUAGE_ELIXIR: 58, + LANGUAGE_FSHARP: 59, + LANGUAGE_LISP: 60, + LANGUAGE_MATLAB: 61, + LANGUAGE_POWERSHELL: 62, + LANGUAGE_SOLIDITY: 63, + LANGUAGE_ADA: 64, + LANGUAGE_OCAML_INTERFACE: 65, + }, }, - "Language": { - "values": { - "LANGUAGE_UNSPECIFIED": 0, - "LANGUAGE_C": 1, - "LANGUAGE_CLOJURE": 2, - "LANGUAGE_COFFEESCRIPT": 3, - "LANGUAGE_CPP": 4, - "LANGUAGE_CSHARP": 5, - "LANGUAGE_CSS": 6, - "LANGUAGE_CUDACPP": 7, - "LANGUAGE_DOCKERFILE": 8, - "LANGUAGE_GO": 9, - "LANGUAGE_GROOVY": 10, - "LANGUAGE_HANDLEBARS": 11, - "LANGUAGE_HASKELL": 12, - "LANGUAGE_HCL": 13, - "LANGUAGE_HTML": 14, - "LANGUAGE_INI": 15, - "LANGUAGE_JAVA": 16, - "LANGUAGE_JAVASCRIPT": 17, - "LANGUAGE_JSON": 18, - "LANGUAGE_JULIA": 19, - "LANGUAGE_KOTLIN": 20, - "LANGUAGE_LATEX": 21, - "LANGUAGE_LESS": 22, - "LANGUAGE_LUA": 23, - "LANGUAGE_MAKEFILE": 24, - "LANGUAGE_MARKDOWN": 25, - "LANGUAGE_OBJECTIVEC": 26, - "LANGUAGE_OBJECTIVECPP": 27, - "LANGUAGE_PERL": 28, - "LANGUAGE_PHP": 29, - "LANGUAGE_PLAINTEXT": 30, - "LANGUAGE_PROTOBUF": 31, - "LANGUAGE_PBTXT": 32, - "LANGUAGE_PYTHON": 33, - "LANGUAGE_R": 34, - "LANGUAGE_RUBY": 35, - "LANGUAGE_RUST": 36, - "LANGUAGE_SASS": 37, - "LANGUAGE_SCALA": 38, - "LANGUAGE_SCSS": 39, - "LANGUAGE_SHELL": 40, - "LANGUAGE_SQL": 41, - "LANGUAGE_STARLARK": 42, - "LANGUAGE_SWIFT": 43, - "LANGUAGE_TSX": 44, - "LANGUAGE_TYPESCRIPT": 45, - "LANGUAGE_VISUALBASIC": 46, - "LANGUAGE_VUE": 47, - "LANGUAGE_XML": 48, - "LANGUAGE_XSL": 49, - "LANGUAGE_YAML": 50, - "LANGUAGE_SVELTE": 51, - "LANGUAGE_TOML": 52, - "LANGUAGE_DART": 53, - "LANGUAGE_RST": 54, - "LANGUAGE_OCAML": 55, - "LANGUAGE_CMAKE": 56, - "LANGUAGE_PASCAL": 57, - "LANGUAGE_ELIXIR": 58, - "LANGUAGE_FSHARP": 59, - "LANGUAGE_LISP": 60, - "LANGUAGE_MATLAB": 61, - "LANGUAGE_POWERSHELL": 62, - "LANGUAGE_SOLIDITY": 63, - "LANGUAGE_ADA": 64, - "LANGUAGE_OCAML_INTERFACE": 65 - } - } - } - } - } - } - } -} + }, + }, + }, + }, + }, +}; diff --git a/packages/web/src/routes/settings-codeium-callback.tsx b/packages/web/src/routes/settings-codeium-callback.tsx index 500261d8..b3f3cb8d 100644 --- a/packages/web/src/routes/settings-codeium-callback.tsx +++ b/packages/web/src/routes/settings-codeium-callback.tsx @@ -16,7 +16,9 @@ export async function exchangeCodeiumAccessTokenForApiKey(accessToken: string): if (!response.ok) { console.error(response); - throw new Error(`Error exchanging codeium access token for api key: ${response.status} ${await response.text()}`); + throw new Error( + `Error exchanging codeium access token for api key: ${response.status} ${await response.text()}`, + ); } return response.json(); @@ -33,44 +35,42 @@ function SettingsCodeiumCallback() { useEffectOnce(() => { if (codeiumApiKey) { - setStatus("already_set"); + setStatus('already_set'); return; } const accessToken = queryParams.get('access_token'); if (!accessToken) { - setStatus("no_access_token"); + setStatus('no_access_token'); return; } - exchangeCodeiumAccessTokenForApiKey(accessToken).then(async response => { - const apiKey = response.api_key; - await updateConfig({ codeiumApiKey: apiKey }); + exchangeCodeiumAccessTokenForApiKey(accessToken) + .then(async (response) => { + const apiKey = response.api_key; + await updateConfig({ codeiumApiKey: apiKey }); - navigate('/settings'); - }).catch(err => { - console.error(err); - setStatus("token_exchange_error"); - }); + navigate('/settings'); + }) + .catch((err) => { + console.error(err); + setStatus('token_exchange_error'); + }); }); switch (status) { - case "loading": + case 'loading': + return
Loading...
; + case 'already_set': + return
Codeium credentials already set!
; + case 'no_access_token': return ( -
Loading...
- ); - case "already_set": - return ( -
Codeium credentials already set!
- ); - case "no_access_token": - return ( -
No access_token query parameter found!
- ); - case "token_exchange_error": - return ( -
Error exchanging codeium access token for api key!
+
+ No access_token query parameter found! +
); + case 'token_exchange_error': + return
Error exchanging codeium access token for api key!
; } -}; +} export default SettingsCodeiumCallback; diff --git a/packages/web/src/routes/settings.tsx b/packages/web/src/routes/settings.tsx index 1657f816..9cc1b59d 100644 --- a/packages/web/src/routes/settings.tsx +++ b/packages/web/src/routes/settings.tsx @@ -220,25 +220,30 @@ function Settings() {
- -
@@ -246,7 +251,9 @@ function Settings() { ) : (
From 317398fcc16251a11786f9aa58e4c35e842acbaa Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 9 Oct 2024 14:14:26 -0400 Subject: [PATCH 09/17] refactor: remove dead code --- packages/web/src/lib/ai-autocomplete/index.ts | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/packages/web/src/lib/ai-autocomplete/index.ts b/packages/web/src/lib/ai-autocomplete/index.ts index f0cba894..7c3019a5 100644 --- a/packages/web/src/lib/ai-autocomplete/index.ts +++ b/packages/web/src/lib/ai-autocomplete/index.ts @@ -47,11 +47,6 @@ export async function runCodiumAiAutocomplete( }), }; - // const verified = GetCompletionsRequest.verify(payload); - // console.log('VERIFIED?', verified); - - // console.log('REQUEST:', payload); - const requestData = GetCompletionsRequest.create(payload); const buffer = GetCompletionsRequest.encode(requestData).finish(); @@ -67,15 +62,9 @@ export async function runCodiumAiAutocomplete( }, }, ); - // console.log('RESPONSE:', response.status); const responseBodyBytes = new Uint8Array(await response.arrayBuffer()); const responseBody = GetCompletionsResponse.decode(responseBodyBytes); - // console.log('RESPONSE COMPLETIONS:'); - // for (const item of responseBody.completionItems) { - // console.log(item.completion.text); - // } - - return responseBody.toJSON(); + return responseBody.toJSON() as CodiumCompletionResult; } From 4c7d0669cc254bc85857b7990a0518dc7832fdd2 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 9 Oct 2024 14:21:10 -0400 Subject: [PATCH 10/17] feat: ensure that codeium ai autocomplete generates suggestions in the right language --- packages/web/src/components/cells/code.tsx | 1 + packages/web/src/lib/ai-autocomplete/index.ts | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/web/src/components/cells/code.tsx b/packages/web/src/components/cells/code.tsx index 0d322671..169d860d 100644 --- a/packages/web/src/components/cells/code.tsx +++ b/packages/web/src/components/cells/code.tsx @@ -388,6 +388,7 @@ export default function ControlledCodeCell(props: Props) { response = await runCodiumAiAutocomplete( codeiumApiKey ?? null, prefix + suffix, + cell.language, prefix.length, ); } catch (err) { diff --git a/packages/web/src/lib/ai-autocomplete/index.ts b/packages/web/src/lib/ai-autocomplete/index.ts index 7c3019a5..02fc7460 100644 --- a/packages/web/src/lib/ai-autocomplete/index.ts +++ b/packages/web/src/lib/ai-autocomplete/index.ts @@ -11,6 +11,7 @@ const EDITOR_API_KEY = 'd49954eb-cfba-4992-980f-d8fb37f0e942'; export async function runCodiumAiAutocomplete( optionalApiKey: string | null, source: string, + sourceLanguage: 'javascript' | 'typescript', cursorOffset: number, ): Promise { const protos = protobuf.Root.fromJSON(languageServerProto as protobuf.INamespace); @@ -36,8 +37,8 @@ export async function runCodiumAiAutocomplete( }), document: DocumentInfo.create({ text: source, - editorLanguage: 'javascript', - language: Language.getOption('JAVASCRIPT'), + editorLanguage: sourceLanguage, + language: Language.getOption(sourceLanguage === 'javascript' ? 'JAVASCRIPT' : 'TYPESCRIPT'), cursorOffset: Long.fromValue(cursorOffset), lineEnding: '\n', }), From 899b1362d68c0bff30f901adc4819166071d8292 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 9 Oct 2024 14:56:15 -0400 Subject: [PATCH 11/17] feat: slightly clean up the codeium oauth interface on the settings page --- packages/web/src/routes/settings.tsx | 81 ++++++++++++++++++---------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/packages/web/src/routes/settings.tsx b/packages/web/src/routes/settings.tsx index 9cc1b59d..d413f5ca 100644 --- a/packages/web/src/routes/settings.tsx +++ b/packages/web/src/routes/settings.tsx @@ -1,4 +1,5 @@ import { useEffect, useState } from 'react'; +import { toast } from 'sonner'; import { CircleCheck, Loader2, CircleX, EyeIcon, EyeOffIcon } from 'lucide-react'; import { Link } from 'react-router-dom'; import { aiHealthcheck, subscribeToMailingList } from '@/lib/server'; @@ -15,7 +16,7 @@ import { Input } from '@srcbook/components/src/components/ui/input'; import useTheme from '@srcbook/components/src/components/use-theme'; import { Switch } from '@srcbook/components/src/components/ui/switch'; import { Button } from '@srcbook/components/src/components/ui/button'; -import { toast } from 'sonner'; +import { cn } from '@/lib/utils'; function Settings() { const { @@ -37,6 +38,8 @@ function Settings() { const [model, setModel] = useState(aiModel); const [baseUrl, setBaseUrl] = useState(aiBaseUrl || ''); const [email, setEmail] = useState(isSubscribed ? subscriptionEmail : ''); + + const [codeiumApiKeyHovering, setCodeiumApiKeyHovering] = useState(false); const [codeiumApiKeyVisible, setCodeiumApiKeyVisible] = useState(false); const updateDefaultLanguage = (value: CodeLanguageType) => { @@ -121,7 +124,7 @@ function Settings() {

AI

-
+
@@ -210,51 +213,75 @@ function Settings() {
)}
-
-
-

AI Autocomplete

+ +

Codeium AI Autocomplete

+
+ By default, Codeium uses a public api token with limited capabilities. Optionally, + sign in to remove rate limits: +
+ {codeiumApiKey ? ( -
-
Codeium API Key:
+
setCodeiumApiKeyHovering(true)} + onMouseLeave={() => setCodeiumApiKeyHovering(false)} + > +
Signed in! Codeium API Key:
- - +
+ + {codeiumApiKeyVisible ? ( + setCodeiumApiKeyVisible(false)} + /> + ) : ( + setCodeiumApiKeyVisible(true)} + /> + )} +
+
) : ( -
-
From 4c9c265eff0ddb6449946292f9565bc1a85e1764 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 9 Oct 2024 14:57:46 -0400 Subject: [PATCH 12/17] docs: add comments on codeium logic --- packages/web/src/lib/ai-autocomplete/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/web/src/lib/ai-autocomplete/index.ts b/packages/web/src/lib/ai-autocomplete/index.ts index 02fc7460..cb1927e8 100644 --- a/packages/web/src/lib/ai-autocomplete/index.ts +++ b/packages/web/src/lib/ai-autocomplete/index.ts @@ -5,9 +5,12 @@ import { type CodiumCompletionResult } from '@srcbook/shared'; import languageServerProto from './language-server-proto'; // NOTE: this EDITOR_API_KEY value was just included as a raw string in -// @codeium/react-code-editor. This seems to not be a secret? +// @codeium/react-code-editor. This seems to not be a secret? See here: +// https://github.com/Exafunction/codeium-react-code-editor/blob/768e1b231c00e078c86bc19c8ede697a1e37ec75/src/components/CodeiumEditor/CompletionProvider.ts#L48 const EDITOR_API_KEY = 'd49954eb-cfba-4992-980f-d8fb37f0e942'; +// NOTE: The below logic has been adapted from codeium's `@codeium/react-code-editor package. See here: +// https://github.com/Exafunction/codeium-react-code-editor/blob/768e1b231c00e078c86bc19c8ede697a1e37ec75/src/components/CodeiumEditor/CompletionProvider.ts#L147-L159 export async function runCodiumAiAutocomplete( optionalApiKey: string | null, source: string, From 49890f14379fd5468b96bf9b3a3b11776ea3cf47 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 9 Oct 2024 15:07:06 -0400 Subject: [PATCH 13/17] refactor: consolodate codeium types with the codeium request code --- packages/shared/index.mts | 1 - packages/shared/src/types/ai-autocomplete.mts | 46 ------------------ packages/web/src/lib/ai-autocomplete/index.ts | 48 ++++++++++++++++++- packages/web/src/lib/server.ts | 1 - 4 files changed, 47 insertions(+), 49 deletions(-) delete mode 100644 packages/shared/src/types/ai-autocomplete.mts diff --git a/packages/shared/index.mts b/packages/shared/index.mts index 6e39bc11..61c47c40 100644 --- a/packages/shared/index.mts +++ b/packages/shared/index.mts @@ -7,6 +7,5 @@ export * from './src/types/cells.mjs'; export * from './src/types/tsserver.mjs'; export * from './src/types/websockets.mjs'; export * from './src/types/secrets.mjs'; -export * from './src/types/ai-autocomplete.mjs'; export * from './src/utils.mjs'; export * from './src/ai.mjs'; diff --git a/packages/shared/src/types/ai-autocomplete.mts b/packages/shared/src/types/ai-autocomplete.mts deleted file mode 100644 index b62fc6ad..00000000 --- a/packages/shared/src/types/ai-autocomplete.mts +++ /dev/null @@ -1,46 +0,0 @@ -export type CodiumCompletionItem = { - completion: { - completionId: string; - text: string; - prefix: string; - stop: string; - score: number; - tokens: Array; - decoded_tokens: Array; - probabilities: Array; - adjustedProbabilities: Array; - generatedLength: string; - }; - completionParts: Array<{ - text: string; - offset: string; - prefix: string; - type: - | 'COMPLETION_PART_TYPE_UNSPECIFIED' - // Single-line completion parts that appear within an existing line of text. - | 'COMPLETION_PART_TYPE_INLINE' - // Possibly multi-line completion parts that appear below an existing line of text. - | 'COMPLETION_PART_TYPE_BLOCK' - // Like COMPLETION_PART_TYPE_INLINE, but overwrites the existing text. - | 'COMPLETION_PART_TYPE_INLINE_MASK'; - }>; - range: { - endOffset: string; - endPosition: { row?: string; col?: string }; - startPosition: { row?: string; col?: string }; - }; -}; - -export type CodiumCompletionResult = { - completionItems?: Array; - state: { - state: - | 'CODEIUM_STATE_UNSPECIFIED' - | 'CODEIUM_STATE_INACTIVE' - | 'CODEIUM_STATE_PROCESSING' - | 'CODEIUM_STATE_SUCCESS' - | 'CODEIUM_STATE_WARNING' - | 'CODEIUM_STATE_ERROR'; - status: string; - }; -}; diff --git a/packages/web/src/lib/ai-autocomplete/index.ts b/packages/web/src/lib/ai-autocomplete/index.ts index cb1927e8..e4b6d530 100644 --- a/packages/web/src/lib/ai-autocomplete/index.ts +++ b/packages/web/src/lib/ai-autocomplete/index.ts @@ -1,6 +1,5 @@ import protobuf from 'protobufjs'; import Long from 'long'; -import { type CodiumCompletionResult } from '@srcbook/shared'; import languageServerProto from './language-server-proto'; @@ -72,3 +71,50 @@ export async function runCodiumAiAutocomplete( return responseBody.toJSON() as CodiumCompletionResult; } + +type CodiumCompletionItem = { + completion: { + completionId: string; + text: string; + prefix: string; + stop: string; + score: number; + tokens: Array; + decoded_tokens: Array; + probabilities: Array; + adjustedProbabilities: Array; + generatedLength: string; + }; + completionParts: Array<{ + text: string; + offset: string; + prefix: string; + type: + | 'COMPLETION_PART_TYPE_UNSPECIFIED' + // Single-line completion parts that appear within an existing line of text. + | 'COMPLETION_PART_TYPE_INLINE' + // Possibly multi-line completion parts that appear below an existing line of text. + | 'COMPLETION_PART_TYPE_BLOCK' + // Like COMPLETION_PART_TYPE_INLINE, but overwrites the existing text. + | 'COMPLETION_PART_TYPE_INLINE_MASK'; + }>; + range: { + endOffset: string; + endPosition: { row?: string; col?: string }; + startPosition: { row?: string; col?: string }; + }; +}; + +type CodiumCompletionResult = { + completionItems?: Array; + state: { + state: + | 'CODEIUM_STATE_UNSPECIFIED' + | 'CODEIUM_STATE_INACTIVE' + | 'CODEIUM_STATE_PROCESSING' + | 'CODEIUM_STATE_SUCCESS' + | 'CODEIUM_STATE_WARNING' + | 'CODEIUM_STATE_ERROR'; + status: string; + }; +}; diff --git a/packages/web/src/lib/server.ts b/packages/web/src/lib/server.ts index f72d4989..0c953e1c 100644 --- a/packages/web/src/lib/server.ts +++ b/packages/web/src/lib/server.ts @@ -4,7 +4,6 @@ import type { MarkdownCellType, CodeCellType, SecretWithAssociatedSessions, - CodiumCompletionResult, } from '@srcbook/shared'; import { SessionType, ExampleSrcbookType } from '@/types'; import SRCBOOK_CONFIG from '@/config'; From 2ed663a482f2f19c6f41c338b1540990c23faed4 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 9 Oct 2024 15:09:00 -0400 Subject: [PATCH 14/17] refactor: reorder imports --- packages/web/src/components/cells/code.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web/src/components/cells/code.tsx b/packages/web/src/components/cells/code.tsx index 169d860d..c8447a61 100644 --- a/packages/web/src/components/cells/code.tsx +++ b/packages/web/src/components/cells/code.tsx @@ -1,5 +1,6 @@ import { useEffect, useState } from 'react'; import { useHotkeys } from 'react-hotkeys-hook'; +import { inlineCopilot } from 'codemirror-copilot'; import { CellType, CodeCellType, @@ -35,7 +36,6 @@ import CodeMirror, { } from '@uiw/react-codemirror'; import useTheme from '@srcbook/components/src/components/use-theme'; import { Dialog, DialogContent } from '@srcbook/components/src/components/ui/dialog'; -import { inlineCopilot } from 'codemirror-copilot'; function tsLinter( cell: CodeCellType, From 5afa2b35c4bb35421604050aa26101079fc18cf9 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 9 Oct 2024 15:17:33 -0400 Subject: [PATCH 15/17] feat: make the codeium-callback page look a little better --- .../src/routes/settings-codeium-callback.tsx | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/web/src/routes/settings-codeium-callback.tsx b/packages/web/src/routes/settings-codeium-callback.tsx index b3f3cb8d..a5f85d0a 100644 --- a/packages/web/src/routes/settings-codeium-callback.tsx +++ b/packages/web/src/routes/settings-codeium-callback.tsx @@ -57,20 +57,29 @@ function SettingsCodeiumCallback() { }); }); + let inner: React.ReactNode; switch (status) { case 'loading': - return
Loading...
; + inner = Loading...; + break; case 'already_set': - return
Codeium credentials already set!
; + inner = Codeium credentials already set!; + break; case 'no_access_token': - return ( + inner = (
No access_token query parameter found!
); + break; case 'token_exchange_error': - return
Error exchanging codeium access token for api key!
; + inner = Error exchanging codeium access token for api key!; + break; } + + return ( +
{inner}
+ ); } export default SettingsCodeiumCallback; From 5656ec5051cce7a65f2392cb62bae60142e6d92c Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 10 Oct 2024 15:33:40 -0400 Subject: [PATCH 16/17] fix: address typo in function name --- packages/web/src/components/cells/code.tsx | 4 ++-- packages/web/src/lib/ai-autocomplete/index.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/web/src/components/cells/code.tsx b/packages/web/src/components/cells/code.tsx index c8447a61..d48b9e8a 100644 --- a/packages/web/src/components/cells/code.tsx +++ b/packages/web/src/components/cells/code.tsx @@ -20,7 +20,7 @@ import { useCells } from '@srcbook/components/src/components/use-cell'; import { mapCMLocationToTsServer, mapTsServerLocationToCM } from './util'; import { toast } from 'sonner'; import { getFileContent } from '@/lib/server'; -import { runCodiumAiAutocomplete } from '@/lib/ai-autocomplete'; +import { runCodeiumAiAutocomplete } from '@/lib/ai-autocomplete'; import { tsHover } from '@/components/cells/hover'; import { autocompletion } from '@codemirror/autocomplete'; import { type Diagnostic, linter } from '@codemirror/lint'; @@ -385,7 +385,7 @@ export default function ControlledCodeCell(props: Props) { inlineCopilot(async (prefix, suffix) => { let response; try { - response = await runCodiumAiAutocomplete( + response = await runCodeiumAiAutocomplete( codeiumApiKey ?? null, prefix + suffix, cell.language, diff --git a/packages/web/src/lib/ai-autocomplete/index.ts b/packages/web/src/lib/ai-autocomplete/index.ts index e4b6d530..a0122f27 100644 --- a/packages/web/src/lib/ai-autocomplete/index.ts +++ b/packages/web/src/lib/ai-autocomplete/index.ts @@ -10,7 +10,7 @@ const EDITOR_API_KEY = 'd49954eb-cfba-4992-980f-d8fb37f0e942'; // NOTE: The below logic has been adapted from codeium's `@codeium/react-code-editor package. See here: // https://github.com/Exafunction/codeium-react-code-editor/blob/768e1b231c00e078c86bc19c8ede697a1e37ec75/src/components/CodeiumEditor/CompletionProvider.ts#L147-L159 -export async function runCodiumAiAutocomplete( +export async function runCodeiumAiAutocomplete( optionalApiKey: string | null, source: string, sourceLanguage: 'javascript' | 'typescript', From d529456435c8fadfe8b72164098f7745c695bf65 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 10 Oct 2024 15:47:25 -0400 Subject: [PATCH 17/17] feat: include other cells as context with ai autocomplete --- packages/web/src/components/cells/code.tsx | 2 ++ packages/web/src/lib/ai-autocomplete/index.ts | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/web/src/components/cells/code.tsx b/packages/web/src/components/cells/code.tsx index d48b9e8a..b729d6fa 100644 --- a/packages/web/src/components/cells/code.tsx +++ b/packages/web/src/components/cells/code.tsx @@ -167,6 +167,7 @@ export default function ControlledCodeCell(props: Props) { ); const { + cells, updateCell: updateCellOnClient, clearOutput, getTsServerDiagnostics, @@ -390,6 +391,7 @@ export default function ControlledCodeCell(props: Props) { prefix + suffix, cell.language, prefix.length, + cells.filter((c): c is CodeCellType => c.type === 'code' && c.id !== cell.id), ); } catch (err) { console.error('Error fetching ai autocomplete suggestion:', err); diff --git a/packages/web/src/lib/ai-autocomplete/index.ts b/packages/web/src/lib/ai-autocomplete/index.ts index a0122f27..61b2ca39 100644 --- a/packages/web/src/lib/ai-autocomplete/index.ts +++ b/packages/web/src/lib/ai-autocomplete/index.ts @@ -1,6 +1,7 @@ import protobuf from 'protobufjs'; import Long from 'long'; +import { CodeCellType } from '@srcbook/shared'; import languageServerProto from './language-server-proto'; // NOTE: this EDITOR_API_KEY value was just included as a raw string in @@ -15,6 +16,7 @@ export async function runCodeiumAiAutocomplete( source: string, sourceLanguage: 'javascript' | 'typescript', cursorOffset: number, + otherCodeCells: Array = [], ): Promise { const protos = protobuf.Root.fromJSON(languageServerProto as protobuf.INamespace); const GetCompletionsRequest = protos.lookupType('exa.language_server_pb.GetCompletionsRequest'); @@ -28,7 +30,17 @@ export async function runCodeiumAiAutocomplete( const apiKey = optionalApiKey ?? EDITOR_API_KEY; const payload = { - otherDocuments: [], + otherDocuments: otherCodeCells.map((otherCodeCell) => + DocumentInfo.create({ + absolutePath: otherCodeCell.filename, + relativePath: otherCodeCell.filename, + text: otherCodeCell.source, + editorLanguage: sourceLanguage, + language: Language.getOption(sourceLanguage === 'javascript' ? 'JAVASCRIPT' : 'TYPESCRIPT'), + cursorOffset: Long.fromValue(0), // NOTE: how do I represent the cursor not being in here? + lineEnding: '\n', + }), + ), metadata: Metadata.create({ ideName: 'web', extensionVersion: '1.0.12',