Merge pull request #1 from feiandxs/develop

added news search

.github/workflows/npmjs.yml

@@ -1,10 +1,11 @@
 name: Publish to NPM
-
 on:
   push:
     branches:
       - main
-
+  pull_request:
+    branches:
+      - main
 jobs:
   publish:
     runs-on: ubuntu-latest
@@ -18,4 +19,4 @@ jobs:
       - run: npm ci
       - run: npm publish
         env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

index.ts

@@ -1,3 +1,4 @@
 import { search } from "./search/search";
+import { searchNews } from "./search/search-news";
 
-export { search };
+export { search, searchNews };

jest.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  testMatch: ['**/__tests__/**/*.js', '**/?(*.)+(spec|test).js'],
+};

main.ts

@@ -1,5 +1,20 @@
-import { getVQD,search } from "./search/search";
+import { search } from "./search/search";
+import { getVQD } from "./search/base";
+import {searchNews} from "./search/search-news";
+
+search('大语言模型微调', {
+  count: 10,
+}).then((res) => {
+  console.log(res);
+  console.log(res.results.length);
+}).catch(console.error);
+
+searchNews('大语言模型微调', {
+  count: 10,
+
+}).then((res) => {
+  console.log(res);
+  console.log(res.results.length);
+}).catch(console.error);
 
-// getVQD('大语言模型').then(console.log).catch(console.error);
 
-search('大语言模型微调').then(console.log).catch(console.error);

package.json

@@ -1,10 +1,10 @@
 {
   "name": "duckduckgogogo",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "duckduckgo api using fetch api",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "jest",
     "dev": "nodemon --watch '**/*.ts' --exec 'ts-node' main.ts"
   },
   "author": "",

readme.md

@@ -1,18 +1,82 @@
 # Duckduckgogogo
 
-[[text](https://www.npmjs.com/package/duckduckgogogo)](<https://www.npmjs.com/package/duckduckgogogo>)
+## Description
+
+This is a library for calling the duckduckgo search engine. It is based on [duck-duck-scrape](https://www.npmjs.com/package/duck-duck-scrape), but the underlying HTTP request is changed from `XMLHttpRequest` to `fetch`, so it can be used in more serverless environments like `cloudflare`.
+
+## 说明
+
+调用 duckduckgo 进行搜索的库，参考了[duck-duck-scrape](https://www.npmjs.com/package/duck-duck-scrape) ，将底层 http 请求调用者由`XMLHttpRequest`换成了 `fetch` ，因而可以在类似 `cloudflare` 等更多的云服务商的 serverless 环境下使用。
+
+对中国用户来说，使用时候需要注意，国内网络不可直接访问 duckduckgo 。
+
+## Source Code
+
+[https://github.com/feiandxs/duckduckgogogo](https://github.com/feiandxs/duckduckgogogo)
+
+## 源码
+
+[https://github.com/feiandxs/duckduckgogogo](https://github.com/feiandxs/duckduckgogogo)
+
+## Available Features
+
+- Search
+  - Regular search
+  - News search
+
+## Todo
+
+- Image search
+- Video search
+- Type Define
+
+## Install
+
+```shell
+npm install duckduckgogogo
+```
+
+or
+
+```shell
+yarn add duckduckgogogo
+```
+
+or
+
+```shell
+pnpm install duckduckgogogo
+```
 
 ## how to use
 
+### web search
+
 ```typescript
 import { search } from 'duckduckgogogo';
 
 const query: string = 'what is the answer to the ultimate question of life, the universe, and everything ?';
 
 const searchResults = await search(query, {
-safeSearch: SafeSearchType.STRICT
+  safeSearch: SafeSearchType.STRICT,
+  count: 10
 });
 
 console.log(searchResults);
 
 ```
+
+### news search
+
+```typescript
+import { searchNews } from 'duckduckgogogo';
+
+const query: string = 'Shanghai Weather'
+
+const searchResults = await searchNews(query, {
+    count: 10
+})
+
+console.log(searchResults);
+
+```

schema/news.schema.ts

@@ -1,3 +1,23 @@
+import { SafeSearchType, SearchTimeType } from "./common.schema";
+
+/** The options for {@link searchNews}. */
+export interface NewsSearchOptions {
+  /** The safe search type of the search. */
+  safeSearch?: SafeSearchType;
+  /** The locale(?) of the search. Defaults to "en-us". */
+  locale?: string;
+  /** The number to offset the results to. */
+  offset?: number;
+  /**
+   * The string that acts like a key to a search.
+   * Set this if you made a search with the same query.
+   */
+  vqd?: string;
+  /** The time range of the articles. */
+  time?: SearchTimeType;
+  count?: number;
+}
+
 /** The news article results from {@link searchNews}. */
 export interface NewsSearchResults {
   /** Whether there were no results found. */

schema/search.schema.ts

@@ -20,6 +20,7 @@ export interface SearchOptions {
    * Set this if you made a search with the same query.
    */
   vqd?: string;
+  count?: number;
 }
 
 export interface CallbackSearchResult {

search/base.ts

@@ -0,0 +1,46 @@
+export const VQD_REGEX = /vqd=['"](\d+-\d+(?:-\d+)?)['"]/;
+
+/**
+ * Get the VQD of a search query.
+ * @param query The query to search
+ * @param ia The type(?) of search
+ * @param options The options of the HTTP request
+ * @returns The VQD
+ */
+export async function getVQD(query: string, ia = 'web', options?: RequestInit): Promise<string> {
+  try {
+    const queryParams = new URLSearchParams({ q: query, ia });
+    const response = await fetch(`https://duckduckgo.com/?${queryParams.toString()}`, options);
+
+    if (!response.ok) {
+      console.log(111)
+      console.log(response.status)
+      throw new Error(`Failed to get the VQD for query "${query}". Status: ${response.status} - ${response.statusText}`);
+    }
+
+    const responseText = await response.text();
+    const vqd = VQD_REGEX.exec(responseText)?.[1];
+    if (!vqd) {
+      throw new Error(`Failed to extract the VQD from the response for query "${query}".`);
+    }
+
+    return vqd;
+  } catch (e: any) {
+    // console.log(e)
+    // console.log(Object.keys(e))
+    // console.log(e.cause)
+    // console.log(Object.keys(e.cause))
+    // console.log('code', e.cause.code)
+    // console.log('message', e.cause.message)
+    // console.log('name', e.cause.name)
+    const err = `Failed to get the VQD for query "${query}".
+      Error: ${e.cause.message}
+    `;
+    throw new Error(err);
+  }
+}
+
+
+export function queryString(query: Record<string, string>) {
+  return new URLSearchParams(query).toString();
+}

search/search-news.ts

@@ -0,0 +1,115 @@
+import {
+  NewsSearchOptions,
+  NewsSearchResults,
+  NewsResult,
+} from '../schema/news.schema';
+
+
+import {
+  SearchTimeType,
+  SafeSearchType
+} from '../schema/common.schema';
+
+import { decode } from 'html-entities';
+
+import{ getVQD, queryString } from './base';
+
+const defaultOptions: NewsSearchOptions = {
+  safeSearch: SafeSearchType.OFF,
+  locale: 'en-us',
+  offset: 0
+};
+
+function sanityCheck(options: NewsSearchOptions) {
+  options = Object.assign({}, defaultOptions, options);
+
+  if (!(options.safeSearch! in SafeSearchType)) throw new TypeError(`${options.safeSearch} is an invalid safe search type!`);
+
+  if (typeof options.safeSearch! === 'string')
+    // @ts-ignore
+    options.safeSearch = SafeSearchType[options.safeSearch!];
+
+  if (typeof options.offset !== 'number') throw new TypeError(`Search offset is not a number!`);
+
+  if (options.offset! < 0) throw new RangeError('Search offset cannot be below zero!');
+
+  if (!options.locale || typeof options.locale! !== 'string') throw new TypeError('Search locale must be a string!');
+
+  if (options.time && !Object.values(SearchTimeType).includes(options.time)) throw new TypeError(`${options.time} is an invalid time filter!`);
+
+  if (options.vqd && !/\d-\d+-\d+/.test(options.vqd)) throw new Error(`${options.vqd} is an invalid VQD!`);
+
+  return options;
+}
+
+
+
+/**
+ * Search news articles.
+ * @category Search
+ * @param query The query to search with
+ * @param options The options of the search
+ * @param needleOptions The options of the HTTP request
+ * @returns Search results
+ */
+export async function searchNews(query: string, options?: NewsSearchOptions): Promise<NewsSearchResults> {
+  if (!query) throw new Error('Query cannot be empty!');
+  if (!options) options = defaultOptions;
+  else options = sanityCheck(options);
+
+  let vqd = options.vqd!;
+  if (!vqd) vqd = await getVQD(query, 'web');
+
+  const queryObject: Record<string, string> = {
+    l: options.locale!,
+    o: 'json',
+    noamp: '1',
+    q: query,
+    vqd,
+    p: options.safeSearch === 0 ? '1' : String(options.safeSearch),
+    df: options.time || '',
+    s: String(options.offset || 0)
+  };
+
+  const response = await fetch(`https://duckduckgo.com/news.js?${queryString(queryObject)}`, {
+    method: 'GET'
+  });
+
+
+
+  if (!response.ok) {
+    throw new Error(`Failed to fetch data from DuckDuckGo. Status: ${response.status} - ${response.statusText}`);
+  }
+
+  const responseBody = await response.text();
+
+
+  if (responseBody.includes('DDG.deep.is506')) {
+    throw new Error('A server error occurred!');
+  }
+
+  if (responseBody.includes('DDG.deep.anomalyDetectionBlock')) {
+    throw new Error('DDG detected an anomaly in the request, you are likely making requests too quickly.');
+  }
+
+
+  const newsResult = JSON.parse(responseBody);
+
+  return {
+    noResults: !newsResult.results.length,
+    vqd,
+    results: (options.count !== undefined
+      ? newsResult.results.slice(0, options.count)
+      : newsResult.results
+    ).map((article: any) => ({
+      date: article.date,
+      excerpt: decode(article.excerpt),
+      image: article.image,
+      relativeTime: article.relative_time,
+      syndicate: article.syndicate,
+      title: decode(article.title),
+      url: article.url,
+      isOld: !!article.is_old
+    })) as NewsResult[]
+  };
+}