fix: 增加并发数控制 & 卡片增加异常图标

xjh22222228 · Jun 28, 2024 · 6a55318 · 6a55318
1 parent df26ad7
commit 6a55318
Show file tree

Hide file tree

Showing 9 changed files with 134 additions and 65 deletions.
diff --git a/scripts/build.mjs b/scripts/build.mjs
@@ -127,35 +127,7 @@ let errorUrlCount = 0
           settings.spiderTitle === 'EMPTY' ||
           settings.spiderTitle === 'ALWAYS'
         ) {
-          const res = await getWebInfo(item.url, { timeout: 3000 })
-          if (!res.status) {
-            console.log(`疑似异常 ${item.url}`)
-          }
-          if (settings.checkUrl) {
-            if (!res.status) {
-              errorUrlCount += 1
-              item.ok = false
-            }
-          }
-          if (res.status) {
-            if (settings.spiderIcon === 'ALWAYS') {
-              item.icon = res.iconUrl
-            } else if (settings.spiderIcon === 'EMPTY' && !item.icon) {
-              item.icon = res.iconUrl
-            }
-
-            if (settings.spiderTitle === 'ALWAYS') {
-              item.name = res.title
-            } else if (settings.spiderTitle === 'EMPTY' && !item.name) {
-              item.name = res.title
-            }
-
-            if (settings.spiderDescription === 'ALWAYS') {
-              item.desc = res.description
-            } else if (settings.spiderDescription === 'EMPTY' && !item.desc) {
-              item.desc = res.description
-            }
-          }
+          items.push(item)
         }
       } else {
         r(item.nav)
@@ -165,40 +137,78 @@ let errorUrlCount = 0
 
   r(db)
 
-  const promises = await Promise.allSettled(
-    items.map((item) => getWebInfo(item.url))
-  )
-
-  for (let i = 0; i < promises.length; i++) {
-    const item = items[i]
-    const res = promises[i].value
-    if (settings.checkUrl) {
-      if (!res.status) {
-        errorUrlCount += 1
-        item.ok = false
-        console.log(`异常 ${item.url}`)
+  console.log('Getting...')
+  const max = settings.spiderQty ?? 20
+  const count = Math.ceil(items.length / max)
+  let current = 0
+  const now = Date.now()
+
+  while (current < count) {
+    const request = []
+    for (let i = current * max; i < current * max + max; i++) {
+      const item = items[i]
+      if (item) {
+        request.push(getWebInfo(item.url, { timeout: 3000 }))
       }
     }
-    if (res.status) {
-      if (settings.spiderIcon === 'ALWAYS') {
-        item.icon = res.iconUrl
-      } else if (settings.spiderIcon === 'EMPTY' && !item.icon) {
-        item.icon = res.iconUrl
-      }
 
-      if (settings.spiderTitle === 'ALWAYS') {
-        item.name = res.title
-      } else if (settings.spiderTitle === 'EMPTY' && !item.name) {
-        item.name = res.title
+    const promises = await Promise.allSettled(request)
+
+    for (let i = 0; i < promises.length; i++) {
+      const idx = current * max + i
+      const item = items[idx]
+      const res = promises[i].value
+      console.log(`${idx}：${res.status ? '正常' : '疑似异常'} ${item.url}`)
+      if (settings.checkUrl) {
+        if (!res.status) {
+          errorUrlCount += 1
+          item.ok = false
+        }
       }
+      if (res.status) {
+        if (settings.spiderIcon === 'ALWAYS') {
+          item.icon = res.iconUrl
+          console.log(
+            `更新图标：${item.url}: "${item.icon}" => "${res.iconUrl}"`
+          )
+        } else if (settings.spiderIcon === 'EMPTY' && !item.icon) {
+          item.icon = res.iconUrl
+          console.log(
+            `更新图标：${item.url}: "${item.icon}" => "${res.iconUrl}"`
+          )
+        }
 
-      if (settings.spiderDescription === 'ALWAYS') {
-        item.desc = res.description
-      } else if (settings.spiderDescription === 'EMPTY' && !item.desc) {
-        item.desc = res.description
+        if (settings.spiderTitle === 'ALWAYS') {
+          console.log(
+            `更新标题：${item.url}: "${item.title}" => "${res.title}"`
+          )
+          item.name = res.title
+        } else if (settings.spiderTitle === 'EMPTY' && !item.name) {
+          console.log(
+            `更新标题：${item.url}: "${item.title}" => "${res.title}"`
+          )
+          item.name = res.title
+        }
+
+        if (settings.spiderDescription === 'ALWAYS') {
+          console.log(
+            `更新描述：${item.url}: "${item.desc}" => "${res.description}"`
+          )
+          item.desc = res.description
+        } else if (settings.spiderDescription === 'EMPTY' && !item.desc) {
+          console.log(
+            `更新描述：${item.url}: "${item.desc}" => "${res.description}"`
+          )
+          item.desc = res.description
+        }
       }
+      console.log('-'.repeat(100))
     }
+    current += 1
   }
+
+  const diff = Math.ceil((Date.now() - now) / 1000)
+  console.log(`Time: ${diff} seconds`)
 })()
 
 process.on('exit', () => {

diff --git a/scripts/internal.mjs b/scripts/internal.mjs
@@ -186,6 +186,7 @@ const TAG_ID_NAME3 = 'Github'
   settings.spiderIcon ??= 'NO'
   settings.spiderDescription ??= 'NO'
   settings.spiderTitle ??= 'NO'
+  settings.spiderQty ??= 20
   fs.writeFileSync(settingsPath, JSON.stringify(settings), {
     encoding: 'utf-8',
   })

diff --git a/src/app/app.module.ts b/src/app/app.module.ts
@@ -84,6 +84,7 @@ import {
   UploadOutline,
   MinusOutline,
   PlusOutline,
+  StopOutline,
 } from '@ant-design/icons-angular/icons'
 
 const icons: IconDefinition[] = [
@@ -95,6 +96,7 @@ const icons: IconDefinition[] = [
   UploadOutline,
   MinusOutline,
   PlusOutline,
+  StopOutline,
 ]
 
 @NgModule({

diff --git a/src/components/card/index.component.html b/src/components/card/index.component.html
@@ -1,6 +1,14 @@
 <!-- 网站 -->
 <ng-container [ngSwitch]="cardStyle">
   <div class="container standard dark-border-color" *ngSwitchCase="'standard'">
+    <span
+      nz-icon
+      nzType="stop"
+      nzTheme="outline"
+      class="stop-icon"
+      title="疑似异常"
+      *ngIf="isLogin && dataSource.ok === false"
+    ></span>
     <div class="rate" *ngIf="settings.showRate">
       <nz-rate
         [ngModel]="dataSource.rate"
@@ -121,6 +129,14 @@
     class="container example2 ellipsis dark-bg dark-white"
     *ngSwitchCase="'example'"
   >
+    <span
+      nz-icon
+      nzType="stop"
+      nzTheme="outline"
+      class="stop-icon"
+      title="疑似异常"
+      *ngIf="isLogin && dataSource.ok === false"
+    ></span>
     <app-logo [src]="dataSource.icon || ''" [name]="dataSource.name"></app-logo>
     <div class="right-box" [title]="dataSource.__desc__ || dataSource.desc">
       <div class="title ellipsis" [innerHTML]="dataSource.name"></div>
@@ -133,6 +149,14 @@
     class="container column2 ellipsis dark-bg dark-white dark-border-color dark-shadow dark-hover"
     *ngSwitchDefault
   >
+    <span
+      nz-icon
+      nzType="stop"
+      nzTheme="outline"
+      class="stop-icon"
+      title="疑似异常"
+      *ngIf="isLogin && dataSource.ok === false"
+    ></span>
     <a [href]="dataSource.url" target="_blank" rel="noreferer noopener">
       <div class="title-bar">
         <app-logo

diff --git a/src/components/card/index.component.scss b/src/components/card/index.component.scss
@@ -10,6 +10,14 @@
     white-space: normal;
     word-break: break-all;
   }
+  .stop-icon {
+    z-index: 3;
+    color: red;
+    font-weight: bold;
+    position: absolute;
+    top: 12px;
+    right: 12px;
+  }
 }
 
 .standard {

diff --git a/src/locale/english.ts b/src/locale/english.ts
@@ -180,7 +180,8 @@ const english: Record<string, string> = {
   _followPage: 'Follow Page',
   _bannerTip:
     'When the height is 0, the width and height of the image will be adaptive, and it is important to ensure that each image has the same size',
-  _checkStatus: 'Check website link status',
+  _checkStatus:
+    'Check website link status(For any of the above updates, it is recommended to enable this, as it takes the same amount of time.)',
   _ignore:
     '(This function is not very useful, it is recommended to ignore or remove it)',
   _standard: 'Standard',
@@ -207,7 +208,11 @@ const english: Record<string, string> = {
   _spiderAlways: 'Always update',
   _spiderStatus: 'Spider Status',
   _spiderTip:
-    'Crawling rules will be triggered every time saving, affecting the construction speed. If necessary, crawling updates will be made. It is recommended to set crawling completion to no crawling',
+    'Crawling rules will be triggered every time saving, affecting the construction speed. Crawling and updating information may be necessary when necessary',
+  _spiderQty: 'Concurrent quantity',
+  _spiderQtyTip: '',
+  _spiderNetTip:
+    'Due to network factors, some websites may not be accessible and can be skipped directly. For reference only',
 }
 
 export default english
diff --git a/src/locale/zh_CN.ts b/src/locale/zh_CN.ts
@@ -172,7 +172,7 @@ const zhCN: Record<string, string> = {
     '通常用于部署自有服务器，当点击保存按钮时会发出GET请求填写的地址',
   _followPage: '跟随页面',
   _bannerTip: '当高度为0时，图片宽高会自适应，务必保证每张图片尺寸一致',
-  _checkStatus: '检查网站链接状态',
+  _checkStatus: '检查网站链接状态(上述任一项更新，建议此开启，耗时相同)',
   _ignore: '（此功能作用不大，建议忽略或删除）',
   _standard: '标准',
   _column: '栏目',
@@ -191,15 +191,18 @@ const zhCN: Record<string, string> = {
   _pendingGet: '正在获取...',
   _logoutAuthCode: '退出授权',
   _spiderRule: '爬取规则',
-  _spiderTitle: '爬取标题',
-  _spiderIcon: '爬取图标',
-  _spiderDesc: '爬取描述',
-  _notSpider: '不爬取',
-  _spiderAlways: '总是抓取',
-  _spiderEmpty: '为空时抓取',
+  _spiderTitle: '更新标题',
+  _spiderIcon: '更新图标',
+  _spiderDesc: '更新描述',
+  _notSpider: '不更新',
+  _spiderAlways: '总是更新',
+  _spiderEmpty: '为空时更新',
   _spiderStatus: '爬取状态',
   _spiderTip:
-    '每次保存时会触发爬取规则，影响构建速度，在必要时进行爬取更新信息，爬取完成建议设置为不爬取',
+    '每次保存时会触发爬取规则，影响构建速度，在必要时进行爬取更新信息',
+  _spiderQty: '并发数量',
+  _spiderQtyTip: '',
+  _spiderNetTip: '由于网络因素，部分网站可能无法访问直接跳过，仅供参考',
 }
 
 export default zhCN
diff --git a/src/types/index.d.ts b/src/types/index.d.ts
@@ -147,6 +147,7 @@ export interface ISettings {
   spiderIcon: Spider
   spiderDescription: Spider
   spiderTitle: Spider
+  spiderQty: number
 }
 
 export interface IConfig {

diff --git a/src/view/system/setting/index.component.html b/src/view/system/setting/index.component.html
@@ -272,6 +272,9 @@ <h2 class="title">{{ $t('_spiderRule') }}</h2>
   <div class="mb10">
     <b>{{ $t('_spiderTip') }}</b>
   </div>
+  <div class="mb10">
+    <b>{{ $t('_spiderNetTip') }}</b>
+  </div>
 
   <nz-form-item>
     <nz-form-label [nzSpan]="4" [nzNoColon]="true">{{
@@ -323,6 +326,18 @@ <h2 class="title">{{ $t('_spiderRule') }}</h2>
     </nz-form-control>
   </nz-form-item>
 
+  <nz-form-item>
+    <nz-form-label [nzSpan]="4">{{ $t('_spiderQty') }}</nz-form-label>
+    <nz-form-control [nzSpan]="10">
+      <nz-slider
+        formControlName="spiderQty"
+        [nzMin]="0"
+        [nzMax]="200"
+      ></nz-slider>
+      <div>{{ $t('_spiderQtyTip') }}</div>
+    </nz-form-control>
+  </nz-form-item>
+
   <div class="divider"></div>
   <h2 class="title">Light {{ $t('_theme') }}</h2>