Skip to content

Commit

Permalink
v4.5.6: fix kanasimi/wikiapi#69
Browse files Browse the repository at this point in the history
  • Loading branch information
@kanasimi
kanasimi committed Dec 21, 2023
1 parent 2b2807d commit 9a1c6bb
Show file tree
Hide file tree
Showing 16 changed files with 293 additions and 144 deletions.
2 changes: 1 addition & 1 deletion _build/build.nodejs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1509,7 +1509,7 @@ function write_qqq_data(resources_path) {
}
// 把 metadata 放在最前面。
message_id_order.unshift("@metadata");
//console.trace(message_id_order.slice(-5));
//console.trace(CeL.string_digest(message_id_order));

let original_contents = CeL.read_file(resources_path + qqq_data_file_name);
const new_contents = JSON.stringify(sort_Object_by_order(qqq_file_data, message_id_order), null, '\t') + '\n';
Expand Down
10 changes: 10 additions & 0 deletions _structure/base.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2564,7 +2564,17 @@ OS='UNIX'; // unknown


var max_log_length = 1000,
prepare_message__string_digest = function (message) {
return library_namespace.string_digest(message, {
slice : max_log_length,
ellipsis : '\n\n...\n\n'
});
};
prepare_message = function (message) {
if (library_namespace.string_digest) {
return (prepare_message = prepare_message__string_digest)(message);
}

message = String(message);
if (message.length > 2 * max_log_length)
message = message.slice(0, max_log_length) + '\n\n...\n\n' + message.slice(-max_log_length);
Expand Down
98 changes: 49 additions & 49 deletions application/locale/resources/i18n/qqq.json
View file

Large diffs are not rendered by default.

98 changes: 49 additions & 49 deletions application/locale/resources/qqq_data.json
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions application/net/Ajax.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1432,8 +1432,8 @@ function module_code(library_namespace) {
// console.trace(options);
// TODO: charset for post_data
to_form_data(post_data, function(data) {
// console.log(data.toString().slice(0,800));
// console.log('>> ' + data.toString().slice(-200));
// library_namespace.data.code.string_digest()
// console.log('>> ' + library_namespace.string_digest(data.toString(), 200));
options.form_data = to_form_data_generated;
get_URL_node(URL_to_fetch, onload, charset, data, options);
}, options.form_data);
Expand Down
2 changes: 2 additions & 0 deletions application/net/wiki/Flow.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
</code>
*
* @since 2019/10/11 拆分自 CeL.application.net.wiki
*
* @see https://www.mediawiki.org/wiki/Structured_Discussions/Deprecation
*/

// More examples: see /_test suite/test.js
Expand Down
2 changes: 2 additions & 0 deletions application/net/wiki/page.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1592,6 +1592,8 @@ function module_code(library_namespace) {
* TODO:
* https://www.mediawiki.org/w/api.php?action=help&modules=searchtranslations
*
* https://www.mediawiki.org/wiki/Extension:Scribunto/Lua_reference_manual#Renaming_or_moving_modules
*
* @example <code>
CeL.wiki.redirect_to('史記', function(redirect_data, page_data) {
Expand Down
9 changes: 4 additions & 5 deletions application/net/wiki/parser/misc.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1938,7 +1938,7 @@ function module_code(library_namespace) {
// prevent patch fieldsep ::= ‘,’ | ‘;’ below
// 必須是在富源乾不會被更動的代碼!
var MARK_as_object = '\0as_object';
// console.trace([ lua_code.slice(0, 800), lua_code.slice(-800) ]);
// console.trace(library_namespace.string_digest(lua_code, 800));

// e.g., parse
// https://raw.githubusercontent.com/wikimedia/mediawiki/master/includes/languages/data/ZhConversion.php
Expand All @@ -1948,8 +1948,7 @@ function module_code(library_namespace) {
// → {"A":"B","C":"D",}
function(all, inner) {
if (false) {
console.trace([ inner.length, inner.slice(0, 800),
inner.slice(-800) ]);
console.trace(library_namespace.string_digest(inner, 800));
}
inner = inner.replace(/("[^"]*")\s*=>\s*("[^"]*")/g, function(all,
from, to) {
Expand Down Expand Up @@ -2056,9 +2055,9 @@ function module_code(library_namespace) {
// --------------------------------------

// Recover MARK_as_object
// console.trace([ lua_code.slice(0, 800), lua_code.slice(-800) ]);
// console.trace(library_namespace.string_digest(lua_code, 800)));
lua_code = lua_code.replace(new RegExp(MARK_as_object, 'g'), '{');
// console.trace([ lua_code.slice(0, 800), lua_code.slice(-800) ]);
// console.trace(library_namespace.string_digest(lua_code, 800)));

// --------------------------------------

Expand Down
6 changes: 3 additions & 3 deletions application/net/wiki/task.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2220,7 +2220,7 @@ function module_code(library_namespace) {
// wiki_API.is_page_data(next[1])
: next[1].title)) {
next[1] = this.last_page;
} else if (this.last_page.id && ('missing' in this.last_page)
} else if (this.last_page && this.last_page.id && ('missing' in this.last_page)
&& get_wikibase_key(this.last_page)
&& get_wikibase_key(this.last_page).title === next[1][1]) {
// 完全還沒設定過 structured data 的檔案是長這樣子:
Expand Down Expand Up @@ -3914,7 +3914,7 @@ function module_code(library_namespace) {
}

callback(data);
});
}, null, options);
};

// 未登錄/anonymous時的token
Expand Down Expand Up @@ -4352,7 +4352,7 @@ function module_code(library_namespace) {
if (typeof callback === 'function') {
callback.call(session, data);
}
});
}, null, session);
};

// --------------------------------------------------------------------------------------------
Expand Down
48 changes: 39 additions & 9 deletions application/net/work_crawler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,10 +270,24 @@ function module_code(library_namespace) {
// console.trace(text_list.length + ' text to be converted.');
if (initializated) {
// 初始化後正常的程序。
var _promise = this.convert_text_language_using(text_list, options)
if (false) {
console.trace('Convert text:', library_namespace
.string_digest(text_list));
}
var _promise = this.convert_text_language_using(text_list, options), has_set_text;
// console.trace(_promise);
// assert: .convert_text_language_using() return thenable
_promise = _promise
//
.then(function set_text_list(converted_text_list) {
// console.trace(converted_text_list);
if (false) {
console.trace('Set converted cache:',
//
library_namespace.string_digest(text_list),
//
library_namespace.string_digest(converted_text_list));
}
has_set_text = true;
text_list.forEach(function(text, index) {
// free
delete _this.converted_text_cache[text].promise;
Expand All @@ -283,10 +297,18 @@ function module_code(library_namespace) {
= converted_text_list[index];
});
// console.trace(_this.converted_text_cache);
if (false) {
return [ text_list, converted_text_list ];
}
});
text_list.forEach(function(text) {
_this.converted_text_cache[text].promise = _promise;
});
if (!has_set_text) {
text_list.forEach(function(text) {
_this.converted_text_cache[text].promise = _promise;
});
}
if (false) {
console.trace(promise, _promise);
}
return promise ? promise.then(_promise) : _promise;
}

Expand All @@ -306,6 +328,9 @@ function module_code(library_namespace) {
= _this.convert_to_language === 'TW'
// library_namespace.extension.zh_conversion.CN_to_TW();
? library_namespace.CN_to_TW : library_namespace.TW_to_CN;
if (false) {
console.trace('cache_converted_text: 初始化完畢。');
}
}).then(cache_converted_text.bind(this, text_list, options));
}

Expand Down Expand Up @@ -333,7 +358,7 @@ function module_code(library_namespace) {
}
console.trace('Delete cache of '
//
+ options.text.slice(0, 40) + '...(' + options.text.length
+ library_namespace.string_digest(options.text)
//
+ ') requiring_thread_count='
//
Expand All @@ -347,8 +372,13 @@ function module_code(library_namespace) {
&& --this.converted_text_cache[options.text]
// 採用 .requiring_thread_count 以避免要求轉換相同文字，後來的取用時已被刪除。
// 若相同操作會呼叫兩次 cache_converted_text()，例如初始化，則此法會出問題。
.requiring_thread_count === 0)
.requiring_thread_count === 0) {
if (false) {
console.trace('clear_converted_text_cache: Free '
+ library_namespace.string_digest(options.text));
}
delete this.converted_text_cache[options.text];
}
} else {
// console.trace(options);
if (false) {
Expand Down Expand Up @@ -396,8 +426,8 @@ function module_code(library_namespace) {
return text;
}

console.log(this.converted_text_cache);
console.log(text);
// console.log(this.converted_text_cache);
console.log(library_namespace.string_digest(text, 200));
console.log(this);
throw new Error(
// 照理不該到這邊。
Expand Down
88 changes: 66 additions & 22 deletions application/net/work_crawler/ebook.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -662,9 +662,15 @@ function module_code(library_namespace) {
// https://www.piaotia.com/html/12/12788/9021489.html 开局奖励一亿条命 第826章 逃得掉吗
<br/><br/>&nbsp;&nbsp;&nbsp;&nbsp;(本章完)<br/><br/>
// https://www.xxbiquke.net/74_74304/39344627.html 仙人只想躺着 第一百七十二章云间临神州，威仪万万里。
<p class="content_detail">
(本章完)
</p>
</code>
*/
text = text.replace(/\(本章完\)(?:<br[^<>]*>|\s)*$/, '');
text = text.replace(
/\(本章完\)(?:<br[^<>]*>|\s)*(<\/p>)?(?:<br[^<>]*>|\s)*$/, '$1');

return text;
}
Expand Down Expand Up @@ -993,7 +999,7 @@ function module_code(library_namespace) {
// console.trace(work_data.convert_options);

data.text = library_namespace.HTML_to_Unicode(
// 處理 HTML tags 以減少其對 this.convert_text_language() 的影響。
// 清理 HTML tags 以減少其對 this.convert_text_language() 的影響。
// TODO: <p> @ qidian.js
library_namespace.EPUB.normailize_contents(data.text
// remove all new-lines
Expand All @@ -1002,6 +1008,11 @@ function module_code(library_namespace) {
.replace(/\s*<br(?:[^\w<>][^<>]*)?>[\r\n]*/ig, '\n')
// .trim()
), true);

data.text = data.text
// 清除最後的 <p></p>
.replace(/(?:<(\w+)[^<>]*>\s*<\/\1>\s*)+$/g, '');

// console.log(data.text);

// return needing to wait language converted
Expand All @@ -1011,19 +1022,61 @@ function module_code(library_namespace) {
work_data.convert_options);
if (promise_of_language_convert) {
if (false) {
library_namespace.log('Convert ' + part_title + '§'
+ chapter_title + ': ' + data.text.slice(0, 40)
+ '...(' + data.text.length + ')');
console.trace('Convert-1 ' + part_title + '§' + chapter_title
+ ': ' + chapter_data.url + '\n'
+ library_namespace.string_digest(data.text));
promise_of_language_convert.chapter_title = chapter_title;
console.log(promise_of_language_convert);
}
library_namespace.log_temporary({
T : [ this.convert_to_language === 'TW'
// gettext_config:{"id":"convert-simplified-chinese-to-traditional-chinese-«$1»"}
? '將簡體中文轉換成繁體中文：《%1》'
// gettext_config:{"id":"convert-traditional-chinese-to-simplified-chinese-«$1»"}
: '将繁体中文转换成简体中文：《%1》',
//
part_title ? part_title + '§' + chapter_title : chapter_title ]
});
process.title = gettext(this.convert_to_language === 'TW'
// gettext_config:{"id":"traditionalize-$1"}
? '繁化: %1'
// gettext_config:{"id":"simplify-$1"}
: '简化: %1', chapter_title) + ' @ ' + this.id;
return ebook.working_promise = ebook.working_promise
//
.then(function() {
return promise_of_language_convert.then(
if (false) {
console.trace('Convert-2 ' + part_title + '§'
+ chapter_title + ': ' + chapter_data.url + '\n'
+ library_namespace.string_digest(data.text));
console.log(promise_of_language_convert);
}

return promise_of_language_convert
/**
* <code>
.then(function(text_list) {
console.trace('Convert-3 '
//
+ part_title + '§' + chapter_title
//
+ ': ' + chapter_data.url + '\n'
//
+ library_namespace.string_digest(data.text));
console.trace(text_list);
})
</code>
*/
.then(
// TODO: 這邊失敗，例如 timeout 的話，會直接跳到最後一章並且出現錯誤。
add_ebook_chapter_actual_work.bind(this, work_data, chapter_NO,
data, options));
}.bind(this));
} else {
if (false) {
console.trace('Add content ' + part_title + '§' + chapter_title
+ ': ' + library_namespace.string_digest(data.text));
}
// 將 ebook 相關作業納入 {Promise}，可保證先添加完章節資料、下載完資源再 pack_ebook()。
return ebook.working_promise = ebook.working_promise
.then(add_ebook_chapter_actual_work.bind(this, work_data,
Expand All @@ -1037,26 +1090,17 @@ function module_code(library_namespace) {

// @see epub_hans_to_hant.js
if (this.convert_to_language) {
if (false) {
console.trace('Get cache of ' + part_title + '§'
+ chapter_title + ': '
+ library_namespace.string_digest(data.text));
}
part_title = this.convert_text_language(part_title);
chapter_title = this.convert_text_language(chapter_title);
library_namespace.log_temporary({
T : [ this.convert_to_language === 'TW'
// gettext_config:{"id":"convert-simplified-chinese-to-traditional-chinese-«$1»"}
? '將簡體中文轉換成繁體中文：《%1》'
// gettext_config:{"id":"convert-traditional-chinese-to-simplified-chinese-«$1»"}
: '将繁体中文转换成简体中文：《%1》', chapter_title ]
});
process.title = gettext(this.convert_to_language === 'TW'
// gettext_config:{"id":"traditionalize-$1"}
? '繁化: %1'
// gettext_config:{"id":"simplify-$1"}
: '简化: %1', chapter_title) + ' @ ' + this.id;
data.original_text = data.text;
if (false) {
library_namespace.log('Get cache of ' + part_title + '§'
+ chapter_title + ': ' + data.text.slice(0, 40)
+ '...(' + data.text.length + ')');
console.trace('→ ' + part_title + '§' + chapter_title);
}
data.original_text = data.text;
data.text = this.convert_text_language(data.text)
// TODO: 把半形標點符號轉換為全形標點符號
.replace(/["'](?:zh-(?:cmn-)?|cmn-)?(?:Hans-)?CN["']/ig,
Expand Down
3 changes: 1 addition & 2 deletions application/net/work_crawler/sites/manhuagui.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,8 +292,7 @@ function module_code(library_namespace) {

// console.log(chapter_list);
// console.log(JSON.stringify(chapter_list));
// console.log(chapter_list.slice(0, 20));
// console.log(chapter_list.slice(-20));
// console.log(library_namespace.string_digest(chapter_list));

// console.log(work_data.chapter_list);
work_data.chapter_list = chapter_list;
Expand Down
Binary file modified ce.js
View file
Binary file not shown.
Loading

0 comments on commit 9a1c6bb

Please sign in to comment.