From 310503ed2719b4eaae02659a18bbb4ac8d4b83a0 Mon Sep 17 00:00:00 2001 From: Mayur Virendra Date: Sun, 7 Apr 2024 04:35:28 +0530 Subject: [PATCH] Updated test cases, added collection detection to action detection function, fixes --- config/openai.json | 7 +- config/registry.json | 15 ++++- controllers/Bot.js | 68 ++++++++++--------- schemas/jsons/actions.js | 13 ++++ services/AI.js | 119 ++++++++++++++++++++++----------- services/Actions.js | 4 +- tests/apis/bot.test.js | 57 ++++++---------- tests/unit/services/ai.test.js | 24 +++++-- 8 files changed, 186 insertions(+), 121 deletions(-) create mode 100644 schemas/jsons/actions.js diff --git a/config/openai.json b/config/openai.json index 37a5879..91d829d 100644 --- a/config/openai.json +++ b/config/openai.json @@ -1,11 +1,12 @@ { "SUPPORTED_ACTIONS": [ - { "action": "search", "description": "Perform a search for a service or product. If a service or product is not specified, its not a search. Listing all bookings is not a search." }, + { "action": "search", "description": "If the user intends to perform a search for a product." }, { "action": "select", "description": "If the user likes or selects any item, this action should be used." }, { "action": "init", "description": "If the user wants to place an order after search and select and has shared the billing details." }, { "action": "confirm", "description": "Confirm an order. This action gets called when users confirms an order." }, - { "action": "clear_chat", "description": "If the user wants to clear the session or restart session or chat." }, - { "action": "clear_all", "description": "If the user wants to clear the complete session or the profile." } + { "action": "clear_chat", "description": "If the user wants to clear the chat." }, + { "action": "clear_all", "description": "If the user wants to clear the session." }, + { "action": "booking_collection", "description": "If the assistant had shared a plan or initinerary and user has indicated to make the bookings for that plan." } ], "SCHEMA_TRANSLATION_CONTEXT": [ { "role": "system", "content": "Your job is to identify the endpoint, method and request body from the given schema, based on the last user input and return the extracted details in the following JSON structure : \n\n {'url':'', 'method':'', 'body':''}'"}, diff --git a/config/registry.json b/config/registry.json index f62fd96..a1fc9d8 100644 --- a/config/registry.json +++ b/config/registry.json @@ -32,7 +32,20 @@ ], "rules": [ "search must have two stops for this domain.", - "Supported stop.type : check-in, check-out" + "Supported stop.type : check-in, check-out", + "fulfillment stops should not have location for this domain." + ] + }, + "tourism": { + "tags": [], + "rules":[ + "Tags and fulfillment should not be used for this domain" + ] + }, + "retail:1.1.0": { + "tags": [], + "rules":[ + "Tags and fulfillment should not be used for this domain" ] } } diff --git a/controllers/Bot.js b/controllers/Bot.js index 9d81b47..ff19e06 100644 --- a/controllers/Bot.js +++ b/controllers/Bot.js @@ -147,36 +147,31 @@ async function process_text(req, res) { session = EMPTY_SESSION; response.formatted = 'Session & profile cleared! You can start a new session now.'; } - else if(ai.action?.action == null) { - let booking_collection_yn = await ai.check_if_booking_collection(message, session.text); - if(booking_collection_yn){ - logger.info(`Booking collection detected: ${booking_collection_yn}`); - - response.formatted = await ai.get_ai_response_to_query('Share the list of bookings to be made? Please include only hotels and tickets to be booked. It should be a short list with just names of bookings to be made. For e.g. Here is a list of bookings you need to make: \n1. hotel at xyz \n2. Tickets for abc \nWhich one do you want to search first?', session.text); - logger.info(`AI response: ${response.formatted}`); - - ai.bookings = await ai.get_bookings_array_from_text(response.formatted); - ai.bookings = ai.bookings.bookings || ai.bookings; - ai.bookings && ai.bookings.map(booking =>{ - booking.transaction_id = uuidv4(); - }) + else if(ai.action?.action === 'booking_collection'){ + response.formatted = await ai.get_ai_response_to_query('Share the list of bookings to be made? Please include only hotels and tickets to be booked. It should be a short list with just names of bookings to be made. For e.g. Here is a list of bookings you need to make: \n1. hotel at xyz \n2. Tickets for abc \nWhich one do you want to search first?', session.text); + logger.info(`AI response: ${response.formatted}`); + + ai.bookings = await ai.get_bookings_array_from_text(response.formatted); + ai.bookings = ai.bookings.bookings || ai.bookings; + ai.bookings && ai.bookings.map(booking =>{ + booking.transaction_id = uuidv4(); + }) - session.text.push({ role: 'user', content: message }); - session.text.push({ role: 'assistant', content: response.formatted }); - } - else{ - // get ai response - response.formatted = await ai.get_ai_response_to_query(message, session.text); - logger.info(`AI response: ${response.formatted}`); - - session.text.push({ role: 'user', content: message }); - session.text.push({ role: 'assistant', content: response.formatted }); - } + session.text.push({ role: 'user', content: message }); + session.text.push({ role: 'assistant', content: response.formatted }); + } + else if(ai.action?.action == null) { + // get ai response + response.formatted = await ai.get_ai_response_to_query(message, session.text); + logger.info(`AI response: ${response.formatted}`); + + session.text.push({ role: 'user', content: message }); + session.text.push({ role: 'assistant', content: response.formatted }); } else{ session.bookings = ai.bookings; - response = await process_action(ai.action, message, session, sender); + response = await process_action(ai.action, message, session, sender, format); ai.bookings = response.bookings; // update actions @@ -225,7 +220,7 @@ async function process_text(req, res) { * @param {*} session * @returns */ -async function process_action(action, text, session, sender=null){ +async function process_action(action, text, session, sender=null, format='application/json'){ let ai = new AI(); let response = { raw: null, @@ -236,7 +231,7 @@ async function process_action(action, text, session, sender=null){ ai.action = action; ai.bookings = session.bookings; - actionsService.send_message(sender, `_Please wait while we process your request through open networks..._`) + format!='application/json' && actionsService.send_message(sender, `_Please wait while we process your request through open networks..._`) // Get schema const schema = await ai.get_schema_by_action(action.action); @@ -249,7 +244,16 @@ async function process_action(action, text, session, sender=null){ if(schema && beckn_context){ let request=null; if(ai.action.action==='search'){ - const message = await ai.get_beckn_message_from_text(text, session.text, beckn_context.domain); + let search_context = [ + ...session.text.slice(-1) + ]; + if(session.profile){ + search_context=[ + { role: 'system', content: `User pforile: ${JSON.stringify(session.profile)}`}, + ...search_context + ] + } + const message = await ai.get_beckn_message_from_text(text, search_context, beckn_context.domain); request = { status: true, data:{ @@ -263,13 +267,13 @@ async function process_action(action, text, session, sender=null){ } } else{ - request = await ai.get_beckn_request_from_text(text, session.actions.raw, beckn_context, schema); + request = await ai.get_beckn_request_from_text(text, [...session.actions.raw.slice(-1)], beckn_context, schema); } if(request.status){ // call api const api_response = await actionsService.call_api(request.data.url, request.data.method, request.data.body, request.data.headers) - actionsService.send_message(sender, `_Your request is processed, generating a response..._`) + format!='application/json' && actionsService.send_message(sender, `_Your request is processed, generating a response..._`) if(!api_response.status){ response.formatted = `Failed to call the API: ${api_response.error}` } @@ -289,7 +293,7 @@ async function process_action(action, text, session, sender=null){ } ai.bookings = response.bookings; - const formatted_response = await ai.get_text_from_json( + const formatted_response = await ai.format_response( api_response.data, [...session.actions.formatted, { role: 'user', content: text }], session.profile @@ -301,7 +305,7 @@ async function process_action(action, text, session, sender=null){ } else{ - response.formatted = "Could not prepare this request. Can you please try something else?" + response.formatted = "Could not process this request. Can you please try something else?" } } diff --git a/schemas/jsons/actions.js b/schemas/jsons/actions.js new file mode 100644 index 0000000..a319983 --- /dev/null +++ b/schemas/jsons/actions.js @@ -0,0 +1,13 @@ +export default { + type: "object", + properties: { + action:{ + type:"string", + description: "action that the user wants to perform. This should be one of th actions defined by supported actions. If its not one of teh actions, its value should be null." + }, + transaction_id:{ + type:"string", + description: "Transaction id of the booking to be performed from the given list of bookings. It should not be set if th action is not from one of the bookings. It shold only be used when the action is 'search'" + } + } +} \ No newline at end of file diff --git a/services/AI.js b/services/AI.js index 7dd48d1..7a5c908 100644 --- a/services/AI.js +++ b/services/AI.js @@ -4,6 +4,7 @@ import logger from '../utils/logger.js' import yaml from 'js-yaml' import { v4 as uuidv4 } from 'uuid' import search from '../schemas/jsons/search.js'; +import actions from '../schemas/jsons/actions.js'; const openai = new OpenAI({ apiKey: process.env.OPENAI_AI_KEY, @@ -25,42 +26,85 @@ class AI { * @param {*} context * @returns */ - async get_beckn_action_from_text(text, context=[], bookings = []){ - let booking_context = []; - if(bookings.length > 0){ - booking_context = [ - { role: 'system', content: `In case of a 'search', the transaction_id should be taken based on which booking out of the list of bookings the user wants to make. For e.g. if the user wants to book the first booking in the list, the transaction_id should be the transaction_id of the first booking. Booking list : ${JSON.stringify(bookings)}`} - ]; - } - const openai_messages = [ - { role: 'system', content: `Your job is to analyse the latest user input and check if it is one of the actions given in the following json with their descriptions : ${JSON.stringify(openai_config.SUPPORTED_ACTIONS)}` }, - { role: 'system', content: `You must return a json response with the following structure : {'action':'SOME_ACTION_OR_NULL', transaction_id: 'SOME_TRANSACTION_ID'}.`}, - ...booking_context, - { role: 'system', content: `Beckn actions must be called in the given order search > select > init > confirm. For e.g. confirm can only be called if init has been called before.`}, - { role: 'system', content: `'action' must be null if its not from the given set of actions. For e.g. planning a trip is not an action. 'find hotels near a place' is a search action.` }, - ...context, - { role: 'user', content: text } - ] + // async get_beckn_action_from_text(text, context=[], bookings = []){ + // let booking_context = [ + // { role: 'system', content: `You must return a json response with the following structure : {'action':'SOME_ACTION_OR_NULL'}`} + // ]; + // if(bookings.length > 0){ + // booking_context = [ + // { role: 'system', content: `You must return a json response with the following structure : {'action':'SOME_ACTION_OR_NULL', 'transaction_id':'TRANSACTION_ID_OF_SELECTED_BOOKING_OR_NULL'}.`}, + // { role: 'system', content: `In case of a 'search', the transaction_id should be taken based on which booking out of the list of bookings the user wants to make. For e.g. if the user wants to book the first booking in the list, the transaction_id should be the transaction_id of the first booking. Booking list : ${JSON.stringify(bookings)}`} + // ]; + // } + // const openai_messages = [ + // { role: 'system', content: `Your job is to analyse the latest user input and check if it is one of the actions given in the following json with their descriptions : ${JSON.stringify(openai_config.SUPPORTED_ACTIONS)}` }, + // ...booking_context, + // { role: 'system', content: `Beckn actions must be called in the given order search > select > init > confirm. For e.g. confirm can only be called if init has been called before.`}, + // { role: 'system', content: `'action' must be null if its not from the given set of actions. For e.g. planning a trip is not an action. 'find hotels near a place' is a search action.` }, + // ...context, + // { role: 'user', content: text } + // ] + // let response = { + // action: null, + // response: null + // } + // try{ + // const completion = await openai.chat.completions.create({ + // messages: openai_messages, + // model: process.env.OPENAI_MODEL_ID, + // temperature: 0, + // response_format: { type: 'json_object' } + // }) + // response = JSON.parse(completion.choices[0].message.content); + // } + // catch(e){ + // logger.error(e); + // } + + // logger.info(`Got action from text : ${JSON.stringify(response)}`) + // return response; + // } + + async get_beckn_action_from_text(instruction, context=[], bookings=[]){ let response = { - action: null, - response: null + action : null } + const messages = [ + { role: 'system', content: `Supported actions : ${JSON.stringify(openai_config.SUPPORTED_ACTIONS)}` }, + { role: 'system', content: `Ongoing bookings : ${JSON.stringify(bookings)}` }, + ...context, + { role: "user", content: instruction } + + ]; + + const tools = [ + { + type: "function", + function: { + name: "get_action", + description: "Identify if the user wants to perform an action.", + parameters: actions + } + } + ]; + try{ - const completion = await openai.chat.completions.create({ - messages: openai_messages, + const gpt_response = await openai.chat.completions.create({ model: process.env.OPENAI_MODEL_ID, - temperature: 0, - response_format: { type: 'json_object' } - }) - response = JSON.parse(completion.choices[0].message.content); + messages: messages, + tools: tools, + tool_choice: "auto", + }); + response = JSON.parse(gpt_response.choices[0].message?.tool_calls[0]?.function?.arguments) || response; + if(!response.action) response.action = null; + logger.info(`Got the action : ${JSON.stringify(response)}`); + return response } catch(e){ logger.error(e); - } - - logger.info(`Got action from text : ${JSON.stringify(response)}`) - return response; + return response; + } } /** @@ -86,7 +130,8 @@ class AI { try{ const completion = await openai.chat.completions.create({ messages: openai_messages, - model: process.env.OPENAI_MODEL_ID + model: process.env.OPENAI_MODEL_ID, + max_tokens: 300 }) response = completion.choices[0].message.content; } @@ -229,7 +274,7 @@ class AI { async get_beckn_message_from_text(instruction, context=[], domain='') { let domain_context = [], policy_context = []; - if(domain_context && domain_context!='') { + if(domain && domain!='') { domain_context = [ { role: 'system', content: `Domain : ${domain}`} ] @@ -243,7 +288,6 @@ class AI { const messages = [ ...policy_context, ...domain_context, - { role: "system", content: "Context goes here..."}, ...context, { role: "user", content: instruction } @@ -317,14 +361,14 @@ class AI { } - async get_text_from_json(json_response, context=[], profile={}) { + async format_response(json_response, context=[], profile={}) { const desired_output = { status: true, message: "" }; let call_to_action = { - 'search': 'You should ask which item the user wants to select to place the order. ', + 'search': 'You should ask which item the user wants to select to place the order. You should show search results in a listing format with important details mentioned such as name, price, rating, location, description or summary etc. and a call to action to select the item.', 'select': 'You should ask if the user wants to initiate the order. You should not use any links from the response.', 'init': 'You should ask if the user wants to confirm the order. ', 'confirm': 'You should display the order id and show the succesful order confirmation message. You should ask if the user wants to book something else.', @@ -335,16 +379,15 @@ class AI { } if(this.bookings.length > 0 && this.action?.action === 'confirm'){ - call_to_action.confirm=`You should display the order id and show the succesful order confirmation message. You should also show the list of pending bookings and ask which booking would they want to do next. Bookings list : ${JSON.stringify(this.bookings)}` + call_to_action.confirm=`You should display the order id and show the succesful order confirmation message. You should also show the list of bookings with theri boooking status and ask which booking would they want to do next. Bookings list : ${JSON.stringify(this.bookings)}` } const openai_messages = [ {role: 'system', content: `Your job is to analyse the input_json and provided chat history to convert the json response into a human readable, less verbose, whatsapp friendly message and return this in a json format as given below: \n ${JSON.stringify(desired_output)}. If the json is invalid or empty, the status in desired output should be false with the relevant error message.`}, - {role: 'system', content: `You should show search results in a listing format with important details mentioned such as name, price, rating, location, description or summary etc. and a call to action to select the item. `}, - {role: 'system', content: `Use this call to action : ${call_to_action[json_response?.context?.action] || ''}`}, + {role: 'system', content: `${call_to_action[json_response?.context?.action] || 'you should ask the user what they want to do next.'}`}, {role: 'system', content: `If the given json looks like an error, summarize the error but for humans, do not include any code or technical details. Produce some user friendly fun messages.`}, - {role: 'system', content: `User pforile : ${JSON.stringify(profile)}`}, + {role: 'system', content: `User profile : ${JSON.stringify(profile)}`}, {role: 'system', content: `Chat history goes next ....`}, - ...context, + ...context.slice(-1), {role: 'assistant',content: `input_json: ${JSON.stringify(json_response)}`}, ] try { diff --git a/services/Actions.js b/services/Actions.js index e7b06c2..134a537 100644 --- a/services/Actions.js +++ b/services/Actions.js @@ -89,11 +89,11 @@ class Actions { // Format the response logger.info(`Formatting response...`); - const get_text_from_json_response = await this.ai.get_text_from_json( + const format_response_response = await this.ai.format_response( call_api_response.data, [...context, { role: 'user', content: message }] ) - response.formatted = get_text_from_json_response.message + response.formatted = format_response_response.message } } } catch (error) { diff --git a/tests/apis/bot.test.js b/tests/apis/bot.test.js index d4ac89c..c4c2719 100644 --- a/tests/apis/bot.test.js +++ b/tests/apis/bot.test.js @@ -145,44 +145,25 @@ describe('Test cases for booking collection', ()=>{ }) it.only('Should return a list of bookings to be made', async ()=>{ - await request(app).post('/webhook').send({ - From: process.env.TEST_RECEPIENT_NUMBER, - Body: "Just bought a new EV - Chevrolet Bolt, thinking of taking it out for a spin with the family.", - }); - - const itinerary = await request(app).post('/webhook').send({ - From: process.env.TEST_RECEPIENT_NUMBER, - Body: "I'm thinking, Yellowstone tomorrow for 3 days, travelling with my family of 4 and a pet. Are you up for it?", - }) - - await request(app).post('/webhook').send({ - From: process.env.TEST_RECEPIENT_NUMBER, - Body: "Can you go ahead and make the bokings?", - }) - - await request(app).post('/webhook').send({ - From: process.env.TEST_RECEPIENT_NUMBER, - Body: "Lets find the hotel first", - }) - - await request(app).post('/webhook').send({ - From: process.env.TEST_RECEPIENT_NUMBER, - Body: "Lets select the first one", - }) - - await request(app).post('/webhook').send({ - From: process.env.TEST_RECEPIENT_NUMBER, - Body: "Mayur Virendra, mayurlibra@gmail.com, 9986949245", - }) - - const response = await request(app).post('/webhook').send({ - From: process.env.TEST_RECEPIENT_NUMBER, - Body: "Lets confirm the order", - }) - - logger.info(JSON.stringify(response.text, null, 2)); - expect(itinerary.status).equal(200); - + const chats = [ + "Hey Alfred, you up? ", + "Just bought a new EV - Chevrolet Bolt, thinking of taking it out for a spin with the family. ", + "I'm thinking, Yellowstone tomorrow for 3 days, traveling with my family of 4 and a pet. Are you up for it? ", + "Perfect, lets make the bookings!", + "Lets find the hotel first", + "Lets select the first one", + "Adam, 9999999999, adam@example.com", + "Lets confirm the order" + ]; + + for(const chat of chats){ + const response = await request(app).post('/webhook').send({ + From: process.env.TEST_RECEPIENT_NUMBER, + Body: chat, + }); + logger.info(JSON.stringify(response.text, null, 2)); + expect(response.status).equal(200); + } }) }) \ No newline at end of file diff --git a/tests/unit/services/ai.test.js b/tests/unit/services/ai.test.js index 110befa..c56e6a7 100644 --- a/tests/unit/services/ai.test.js +++ b/tests/unit/services/ai.test.js @@ -87,16 +87,26 @@ describe('Test cases for services/ai/get_beckn_action_from_text()', () => { }); it('Should return `clear_chat` action when user wishes to clear the chat', async () => { - const response = await ai.get_beckn_action_from_text('Can you clear this session ', hotel_session.data.actions.formatted); + const response = await ai.get_beckn_action_from_text('Can you clear my chat?', hotel_session.data.actions.formatted); expect(response).to.have.property('action') expect(response.action).to.equal('clear_chat'); }); it('Should return `clear_all` action when user wishes to clear the the entire session including profile.', async () => { - const response = await ai.get_beckn_action_from_text('Can you clear this session along with my profile.', hotel_session.data.actions.formatted); + const response = await ai.get_beckn_action_from_text('Can you clear my session?', hotel_session.data.actions.formatted); expect(response).to.have.property('action') expect(response.action).to.equal('clear_all'); }); + + it('Should return `booking_collection` action If the user wants to make multiple bookings', async () => { + const context = [ + {role: 'user', content: trip_planning.TRIP_DETAILS}, + {role: 'assistant', content: trip_planning.TRIP_DETAILS_RESPONSE} + ]; + const response = await ai.get_beckn_action_from_text('Perfect! can you make the bookings?', context); + expect(response).to.have.property('action') + expect(response.action).to.equal('booking_collection'); + }); }) describe('Test cases for get_ai_response_to_query() function', () => { @@ -295,16 +305,16 @@ describe('Test cases for services/ai/get_beckn_request_from_text()', () => { }); -describe('Test cases for services/ai/get_text_from_json()', () => { - it('Should test get_text_from_json() and throw response with success false for empty object', async () => { - const response = await ai.get_text_from_json({}) +describe('Test cases for services/ai/format_response()', () => { + it('Should test format_response() and throw response with success false for empty object', async () => { + const response = await ai.format_response({}) expect(response.status).to.equal(false) }) - it('Should test get_text_from_json() return some message with success true', async () => { + it('Should test format_response() return some message with success true', async () => { const context = [ {role: 'user', content: 'I want to search for some ev chargers'} ] - const response = await ai.get_text_from_json(on_search, context) + const response = await ai.format_response(on_search, context) expect(response.status).to.equal(true) }) })