Merge pull request #43 from YDX-2147483647/YDX-2147483647/issue42

src/examples/cli.ts

@@ -1,9 +1,11 @@
 import { hook, update_notices } from '../core/index.js'
 import { add_hook } from '../plugin/cli/index.js'
+import add_normalize_hook from '../plugin/normalize/index.js'
 
 add_hook.verbose(hook)
 add_hook.preview_output(hook)
 add_hook.progress_bar(hook)
 add_hook.recent_filter(hook, 90)
+add_normalize_hook(hook)
 
 await update_notices()

src/examples/ding-cli.ts

@@ -5,10 +5,12 @@
 import { hook, update_notices } from '../core/index.js'
 import { add_hook } from '../plugin/cli/index.js'
 import robot from '../plugin/ding/index.js'
+import add_normalize_hook from '../plugin/normalize/index.js'
 import { logger } from '../util/logger.js'
 
 add_hook.verbose(hook)
 add_hook.progress_bar(hook)
+add_normalize_hook(hook)
 
 const { all_notices, new_notices, change } = await update_notices()
 

src/examples/proxy-cli.ts

@@ -1,10 +1,12 @@
 import { hook, update_notices } from '../core/index.js'
 import { add_hook } from '../plugin/cli/index.js'
 import add_proxy_hook from '../plugin/proxy/index.js'
+import add_normalize_hook from '../plugin/normalize/index.js'
 
 add_hook.verbose(hook)
 add_hook.preview_output(hook)
 add_hook.progress_bar(hook)
 add_proxy_hook(hook)
+add_normalize_hook(hook)
 
 await update_notices()

src/examples/rss-cli.ts

@@ -1,10 +1,12 @@
 import { hook, update_notices } from '../core/index.js'
 import { add_hook } from '../plugin/cli/index.js'
+import add_normalize_hook from '../plugin/normalize/index.js'
 import add_rss_hook from '../plugin/rss/index.js'
 
 add_hook.verbose(hook)
 add_hook.progress_bar(hook)
 add_hook.recent_filter(hook, 90)
+add_normalize_hook(hook)
 add_rss_hook(hook)
 
 await update_notices()

src/examples/server-cli.ts

@@ -5,11 +5,13 @@
 import { hook, update_notices } from '../core/index.js'
 import { add_hook } from '../plugin/cli/index.js'
 import add_proxy_hook from '../plugin/proxy/index.js'
+import add_normalize_hook from '../plugin/normalize/index.js'
 import add_rss_hook from '../plugin/rss/index.js'
 
 add_hook.verbose(hook)
 add_hook.progress_bar(hook)
 add_proxy_hook(hook)
+add_normalize_hook(hook)
 add_rss_hook(hook)
 
 await update_notices()

src/examples/server-ding-cli.ts

@@ -6,12 +6,14 @@ import { hook, update_notices } from '../core/index.js'
 import { add_hook } from '../plugin/cli/index.js'
 import robot from '../plugin/ding/index.js'
 import add_proxy_hook from '../plugin/proxy/index.js'
+import add_normalize_hook from '../plugin/normalize/index.js'
 import add_rss_hook from '../plugin/rss/index.js'
 import { logger } from '../util/logger.js'
 
 add_hook.verbose(hook)
 add_hook.progress_bar(hook)
 add_proxy_hook(hook)
+add_normalize_hook(hook)
 add_rss_hook(hook)
 
 const { new_notices, change } = await update_notices()

src/plugin/normalize/index.ts

@@ -0,0 +1,16 @@
+/**
+ * 重整不完整的标题
+ * @module
+ */
+
+import { type HookCollectionType } from '../../core/index.js'
+import { normalize } from './normalize.js'
+
+/**
+ * `fetch_each`后重整不完整的标题
+ */
+export default function add_normalize_hook (hook: HookCollectionType) {
+    hook.after('fetch_each', async ({ notices }) => {
+        notices.forEach(n => { n.title = normalize(n.title) })
+    })
+}

src/plugin/normalize/normalize.test.ts

@@ -0,0 +1,41 @@
+import { assert } from 'chai'
+import { describe, it } from 'mocha'
+
+import { normalize } from './normalize.js'
+
+function assert_normalize (raw: string, expected: string) {
+    assert.equal(normalize(raw), expected)
+}
+
+describe('重整标题', () => {
+    it('保持完整标题', () => {
+        assert_normalize(
+            '2023年北京市成人高校招生录取信息与分数线查询',
+            '2023年北京市成人高校招生录取信息与分数线查询',
+        )
+    })
+    it('修正截断标题的省略号', () => {
+        assert_normalize(
+            '【通知】明德书院关于2023年社会捐助类奖助学金（部分）评选工...',
+            '【通知】明德书院关于2023年社会捐助类奖助学金（部分）评选工……',
+        )
+    })
+    it('重整残余特殊字符的标题', () => {
+        assert_normalize(
+            '北理工宇航学院2024年接收优秀应届本科毕业生推荐免试研究生&#...',
+            '北理工宇航学院2024年接收优秀应届本科毕业生推荐免试研究生……',
+        )
+        assert_normalize(
+            '光电学院2024年接收优秀应届本科毕业生推荐免试研究生（含本直博生ÿ...',
+            '光电学院2024年接收优秀应届本科毕业生推荐免试研究生（含本直博生……',
+        )
+        assert_normalize(
+            '北京理工大学管理与经济学院2024年接收优秀应届本科毕业生推荐免试攻读研究生（含本直博生\u000f...',
+            '北京理工大学管理与经济学院2024年接收优秀应届本科毕业生推荐免试攻读研究生（含本直博生……',
+        )
+        assert_normalize(
+            '关于2023年社会捐助类奖助学金（部分࿰...',
+            '关于2023年社会捐助类奖助学金（部分……',
+        )
+    })
+})

src/plugin/normalize/normalize.ts

@@ -0,0 +1,13 @@
+/**
+ * 重整不完整的文字
+ * @param text 原始文字
+ * @returns 重整化的文字
+ */
+export function normalize (text: string) {
+    // 保留完整标题
+    if (!text.endsWith('...')) {
+        return text
+    }
+
+    return text.replace(/[^A-Za-z\u4E00-\u9FFF]*\.{3}$/, '……')
+}