From 205fe1d5e874c77ec3f2386980b8ff50c263f9c3 Mon Sep 17 00:00:00 2001 From: Versun <2398708+versun@users.noreply.github.com> Date: Tue, 9 Jul 2024 01:02:20 +0000 Subject: [PATCH] Next (#91) --- .github/workflows/latest-docker-image.yml | 2 +- core/actions.py | 257 ++++++++++++ core/admin.py | 487 ++-------------------- core/custom_admin_site.py | 133 ++++++ core/forms.py | 97 +++++ core/inlines.py | 69 +++ core/tasks.py | 35 +- core/urls.py | 1 + dev-diaries/versun-dev-diary.md | 24 ++ templates/admin/base_site.html | 110 +++-- translator/models/base.py | 2 +- translator/models/deeplweb.py | 7 +- translator/models/deeplx.py | 7 +- translator/models/doubao.py | 3 +- translator/models/google_translate_web.py | 7 +- utils/modelAdmin_utils.py | 138 ------ utils/text_handler.py | 1 + 17 files changed, 718 insertions(+), 662 deletions(-) create mode 100644 core/actions.py create mode 100644 core/custom_admin_site.py create mode 100644 core/forms.py create mode 100644 core/inlines.py create mode 100644 dev-diaries/versun-dev-diary.md diff --git a/.github/workflows/latest-docker-image.yml b/.github/workflows/latest-docker-image.yml index 27c16ff..b97b8f0 100644 --- a/.github/workflows/latest-docker-image.yml +++ b/.github/workflows/latest-docker-image.yml @@ -2,7 +2,7 @@ name: latest:Docker Image CI on: release: - types: [published] + types: [released] branches: - main diff --git a/core/actions.py b/core/actions.py new file mode 100644 index 0000000..fcbefeb --- /dev/null +++ b/core/actions.py @@ -0,0 +1,257 @@ +import logging +from datetime import datetime +from ast import literal_eval +from opyml import OPML, Outline, Head +from huey.contrib.djhuey import HUEY as huey + +from django.contrib import admin +from django.shortcuts import render, redirect +from django.urls import reverse +from django.http import HttpResponse +from django.db import transaction +from django.utils.translation import gettext_lazy as _ + +from utils.modelAdmin_utils import get_translator_and_summary_choices +from .custom_admin_site import core_admin_site +from .models import O_Feed +from .tasks import update_original_feed, update_translated_feed + + +@admin.display(description=_("Export selected feeds as OPML")) +def o_feed_export_as_opml(modeladmin, request, queryset): + try: + opml_obj = OPML() + opml_obj.head = Head( + title="Original Feeds | RSS Translator", + date_created=datetime.now().strftime("%a, %d %b %Y %H:%M:%S %z"), + owner_name="RSS Translator", + ) + + categories = {} + for item in queryset: + category = item.category.name if item.category else "default" + # category_outline = Outline(text=category) + if category not in categories: + categories[category] = Outline(text=category) + + item_outline = Outline( + title=item.name, + text=item.name, + type="rss", + xml_url=item.feed_url, + html_url=item.feed_url, + ) + categories[category].outlines.append(item_outline) + + # category_outline.outlines.append(item_outline) + # opml_obj.body.outlines.append(category_outline) + for category_outline in categories.values(): + opml_obj.body.outlines.append(category_outline) + + response = HttpResponse(opml_obj.to_xml(), content_type="application/xml") + response["Content-Disposition"] = ( + 'attachment; filename="rsstranslator_original_feeds.opml"' + ) + return response + except Exception as e: + logging.error("o_feed_export_as_opml: %s", str(e)) + return HttpResponse("An error occurred", status=500) + + +@admin.display(description=_("Export selected feeds as OPML")) +def t_feed_export_as_opml(modeladmin, request, queryset): + try: + opml_obj = OPML() + opml_obj.head = Head( + title="Translated Feeds | RSS Translator", + date_created=datetime.now().strftime("%a, %d %b %Y %H:%M:%S %z"), + owner_name="RSS Translator", + ) + + categories = {} + for item in queryset: + category = item.o_feed.category.name if item.o_feed.category else "default" + text = item.o_feed.name or "No Name" + xml_url = request.build_absolute_uri( + reverse("core:rss", kwargs={"feed_sid": item.sid}) + ) + + if category not in categories: + categories[category] = Outline(text=category) + + item_outline = Outline( + title=text, + text=text, + type="rss", + xml_url=xml_url, + html_url=item.o_feed.feed_url, + ) + categories[category].outlines.append(item_outline) + for category_outline in categories.values(): + opml_obj.body.outlines.append(category_outline) + + response = HttpResponse(opml_obj.to_xml(), content_type="application/xml") + response["Content-Disposition"] = ( + 'attachment; filename="rsstranslator_translated_feeds.opml"' + ) + return response + except Exception as e: + logging.error("t_feed_export_as_opml: %s", str(e)) + return HttpResponse("An error occurred", status=500) + + +@admin.display(description=_("Force update")) +def o_feed_force_update(modeladmin, request, queryset): + logging.info("Call o_feed_force_update: %s", queryset) + with transaction.atomic(): + for instance in queryset: + instance.etag = "" + instance.valid = None + instance.save() + #logging.info("Call revoke_tasks_by_arg in o_feed_force_update") + #revoke_tasks_by_arg(instance.sid) + update_original_feed.schedule( + args=(instance.sid,True), delay=1, + ) # 会执行一次save() + + +@admin.display(description=_("Force update")) +def t_feed_force_update(modeladmin, request, queryset): + logging.info("Call t_feed_force_update: %s", queryset) + with transaction.atomic(): + for instance in queryset: + instance.modified = None + instance.status = None + instance.save() + #logging.info("Call revoke_tasks_by_arg in o_feed_force_update") + #revoke_tasks_by_arg(instance.sid) #will check in update_translated_feed task + update_translated_feed.schedule( + args=(instance.sid,True), delay=1 + ) # 会执行一次save() + + +@admin.display(description=_("Batch modification")) +def o_feed_batch_modify(modeladmin, request, queryset): + if "apply" in request.POST: + logging.info("Apply o_feed_batch_modify") + post_data = request.POST + fields = { + "update_frequency": "update_frequency_value", + "max_posts": "max_posts_value", + "translator": "translator_value", + "translation_display": "translation_display_value", + "summary_engine": "summary_engine_value", + "summary_detail": "summary_detail_value", + "additional_prompt": "additional_prompt_value", + "fetch_article": "fetch_article", + "quality": "quality", + "category": "category_value", + } + field_types = { + "update_frequency": int, + "max_posts": int, + "translation_display": int, + "summary_detail": float, + "additional_prompt": str, + "fetch_article": literal_eval, + "quality": literal_eval, + } + update_fields = {} + # tags_value = None + + for field, value_field in fields.items(): + value = post_data.get(value_field) + if post_data.get(field, "Keep") != "Keep" and value: + match field: + case "translator": + content_type_id, object_id = map(int, value.split(":")) + update_fields["content_type_id"] = content_type_id + update_fields["object_id"] = object_id + case "summary_engine": + content_type_summary_id, object_id_summary = map( + int, value.split(":") + ) + update_fields["content_type_summary_id"] = ( + content_type_summary_id + ) + update_fields["object_id_summary"] = object_id_summary + case "category": + tag_model = O_Feed.category.tag_model + category_o, _ = tag_model.objects.get_or_create(name=value) + update_fields["category"] = category_o + + case _: + update_fields[field] = field_types.get(field, str)(value) + + if update_fields: + queryset.update(**update_fields) + # for obj in queryset: + # obj.category.update_count() + + # if tags_value is not None: + # for obj in queryset: + # obj.tags = [*tags_value] + # obj.save() + # O_Feed.objects.bulk_update(queryset, ['tags'])?? + + # self.message_user(request, f"Successfully modified {queryset.count()} items.") + # return HttpResponseRedirect(request.get_full_path()) + return redirect(request.get_full_path()) + + translator_choices, summary_engine_choices = get_translator_and_summary_choices() + logging.info( + "translator_choices: %s, summary_engine_choices: %s", + translator_choices, + summary_engine_choices, + ) + return render( + request, + "admin/o_feed_batch_modify.html", + context={ + **core_admin_site.each_context(request), + "items": queryset, + "translator_choices": translator_choices, + "summary_engine_choices": summary_engine_choices, + }, + ) + + +@admin.display(description=_("Batch modification")) +def t_feed_batch_modify(modeladmin, request, queryset): + if "apply" in request.POST: + logging.info("Apply t_feed_batch_modify") + translate_title = request.POST.get("translate_title", "Keep") + translate_content = request.POST.get("translate_content", "Keep") + summary = request.POST.get("summary", "Keep") + match translate_title: + case "Keep": + pass + case "True": + queryset.update(translate_title=True) + case "False": + queryset.update(translate_title=False) + + match translate_content: + case "Keep": + pass + case "True": + queryset.update(translate_content=True) + case "False": + queryset.update(translate_content=False) + + match summary: + case "Keep": + pass + case "True": + queryset.update(summary=True) + case "False": + queryset.update(summary=False) + + # self.message_user(request, f"Successfully modified {queryset.count()} items.") + # return HttpResponseRedirect(request.get_full_path()) + return redirect(request.get_full_path()) + return render( + request, + "admin/t_feed_batch_modify.html", + context={**core_admin_site.each_context(request), "items": queryset}, + ) diff --git a/core/admin.py b/core/admin.py index 9513fdb..37f9785 100644 --- a/core/admin.py +++ b/core/admin.py @@ -1,315 +1,27 @@ import logging -from ast import literal_eval -from django import forms from django.contrib import admin from django.conf import settings -from django.shortcuts import render -from django.urls import path -from django.shortcuts import redirect -from django.core.paginator import Paginator from django.contrib.auth.models import User, Group -from django.urls import reverse from django.utils.html import format_html -from django.utils.http import url_has_allowed_host_and_scheme from django.utils.translation import gettext_lazy as _ -from .models import O_Feed, T_Feed -# from taggit.models import Tag -from .tasks import update_original_feed, update_translated_feed -from utils.modelAdmin_utils import ( - CustomModelActions, - get_translator_and_summary_choices, - get_all_app_models, - valid_icon, +from .models import O_Feed, T_Feed +from .custom_admin_site import core_admin_site +from .forms import O_FeedForm +from .inlines import T_FeedInline +from .actions import ( + o_feed_export_as_opml, + t_feed_export_as_opml, + o_feed_force_update, + t_feed_force_update, + o_feed_batch_modify, + t_feed_batch_modify, ) +from .tasks import update_original_feed, update_translated_feed +from utils.modelAdmin_utils import valid_icon -class CoreAdminSite(admin.AdminSite): - site_header = _("RSS Translator Admin") - site_title = _("RSS Translator") - index_title = _("Dashboard") - - def get_urls(self): - urls = super().get_urls() - custom_urls = [ - path("translator/add", translator_add_view, name="translator_add"), - path("translator/list", translator_list_view, name="translator_list"), - ] - return custom_urls + urls - - def get_app_list(self, request, app_label=None): - app_list = super().get_app_list(request, app_label) - app_list += [ - { - "name": _("Engine"), - "app_label": "engine", - "models": [ - { - "name": _("Translator"), - "object_name": "Translator", - "admin_url": "/translator/list", - "add_url": "/translator/add", - # "view_only": False, - } - ], - } - ] - - return app_list - - -core_admin_site = CoreAdminSite() - - -class TranslatorPaginator(Paginator): - def __init__(self): - super().__init__(self, 100) - - self.translator_count = len(get_all_app_models("translator")) - - @property - def count(self): - return self.translator_count - - def page(self, number): - limit = self.per_page - offset = (number - 1) * self.per_page - return self._get_page( - self.enqueued_items(limit, offset), - number, - self, - ) - - # Copied from Huey's SqliteStorage with some modifications to allow pagination - def enqueued_items(self, limit, offset): - translators = get_all_app_models("translator") - translator_list = [] - for model in translators: - objects = ( - model.objects.all() - .order_by("name") - .values_list("id", "name", "valid")[offset : offset + limit] - ) - for obj_id, obj_name, obj_valid in objects: - translator_list.append( - { - "id": obj_id, - "table_name": model._meta.db_table.split("_")[1], - "name": obj_name, - "valid": valid_icon(obj_valid), - "provider": model._meta.verbose_name, - } - ) - - return translator_list - - -def translator_list_view(request): - page_number = int(request.GET.get("p", 1)) - paginator = TranslatorPaginator() - page = paginator.get_page(page_number) - page_range = paginator.get_elided_page_range(page_number, on_each_side=2, on_ends=2) - - context = { - **core_admin_site.each_context(request), - "title": "Translator", - "page": page, - "page_range": page_range, - "translators": page.object_list, - } - return render(request, "admin/translator.html", context) - - -def translator_add_view(request): - if request.method == "POST": - translator_name = request.POST.get("translator_name", "/") - # redirect to example.com/translator/translator_name/add - target = f"/translator/{translator_name}/add" - return ( - redirect(target) - if url_has_allowed_host_and_scheme(target, allowed_hosts=None) - else redirect("/") - ) - else: - models = get_all_app_models("translator") - translator_list = [] - for model in models: - translator_list.append( - { - "table_name": model._meta.db_table.split("_")[1], - "provider": model._meta.verbose_name, - } - ) - context = { - **core_admin_site.each_context(request), - "translator_choices": translator_list, - } - return render(request, "admin/translator_add.html", context) - - -class T_FeedForm(forms.ModelForm): - class Meta: - model = T_Feed - fields = ["language", "translate_title", "sid"] - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.fields["sid"].required = False - if self.instance.pk: - self.fields["language"].disabled = True - self.fields["sid"].disabled = True - - -class T_FeedInline(admin.TabularInline): - model = T_Feed - form = T_FeedForm - fields = [ - "language", - "obj_status", - "feed_url", - "translate_title", - "translate_content", - "summary", - "total_tokens", - "total_characters", - "size_in_kb", - "sid", - ] - readonly_fields = ( - "feed_url", - "obj_status", - "size_in_kb", - "total_tokens", - "total_characters", - ) - extra = 1 - - def feed_url(self, obj): - if obj.sid: - rss = reverse("core:rss", kwargs={"feed_sid": obj.sid}) - rss_url = self.request.build_absolute_uri(rss) - json = reverse("core:json", kwargs={"feed_sid": obj.sid}) - json_url = self.request.build_absolute_uri(json) - return format_html( - "RSS " - " | " - "JSON " - "", - rss_url, - json_url, - ) - return "" - - feed_url.short_description = _("Translated Feed URL") - - class Media: - js = ("js/admin/copytoclipboard.js",) - - def size_in_kb(self, obj): - return int(obj.size / 1024) - - size_in_kb.short_description = _("Size(KB)") - - def obj_status(self, obj): - if not obj.pk: - return "" - return valid_icon(obj.status) - - obj_status.short_description = _("Status") - - def get_formset(self, request, obj=None, **kwargs): - # Store the request for use in feed_url - self.request = request - return super(T_FeedInline, self).get_formset(request, obj, **kwargs) - - -class O_FeedForm(forms.ModelForm): - # 自定义字段，使用ChoiceField生成下拉菜单 - translator = forms.ChoiceField( - choices=(), - required=False, - help_text=_("Select a valid translator"), - label=_("Translator"), - ) - summary_engine = forms.ChoiceField( - choices=(), - required=False, - help_text=_("Select a valid AI engine"), - label=_("Summary Engine"), - ) - - def __init__(self, *args, **kwargs): - super(O_FeedForm, self).__init__(*args, **kwargs) - - self.fields["translator"].choices, self.fields["summary_engine"].choices = ( - get_translator_and_summary_choices() - ) - - # 如果已经有关联的对象，设置默认值 - instance = getattr(self, "instance", None) - if instance and instance.pk and instance.content_type and instance.object_id: - self.fields[ - "translator" - ].initial = f"{instance.content_type.id}:{instance.object_id}" - - if ( - instance - and instance.pk - and instance.content_type_summary - and instance.object_id_summary - ): - self.fields[ - "summary_engine" - ].initial = ( - f"{instance.content_type_summary.id}:{instance.object_id_summary}" - ) - - # self.fields['translator'].short_description = _("Translator") - - class Meta: - model = O_Feed - fields = [ - "feed_url", - "update_frequency", - "max_posts", - "translator", - "translation_display", - "summary_engine", - "summary_detail", - "additional_prompt", - "fetch_article", - "quality", - "name", - "category", - ] - - # 重写save方法，以处理自定义字段的数据 - def save(self, commit=True): - # 获取选择的translator，并设置content_type和translator_object_id - if self.cleaned_data["translator"]: - content_type_id, object_id = map( - int, self.cleaned_data["translator"].split(":") - ) - self.instance.content_type_id = content_type_id - self.instance.object_id = object_id - else: - self.instance.content_type = None - self.instance.object_id = None - - if self.cleaned_data["summary_engine"]: - content_type_summary_id, object_id_summary = map( - int, self.cleaned_data["summary_engine"].split(":") - ) - self.instance.content_type_summary_id = content_type_summary_id - self.instance.object_id_summary = object_id_summary - else: - self.instance.content_type_summary_id = None - self.instance.object_id_summary = None - - return super(O_FeedForm, self).save(commit=commit) - - -class O_FeedAdmin(admin.ModelAdmin, CustomModelActions): +class O_FeedAdmin(admin.ModelAdmin): form = O_FeedForm inlines = [T_FeedInline] list_display = [ @@ -325,7 +37,8 @@ class O_FeedAdmin(admin.ModelAdmin, CustomModelActions): ] search_fields = ["name", "feed_url", "category__name"] list_filter = ["valid", "category"] - actions = ["o_feed_force_update", "o_feed_export_as_opml", "o_feed_batch_modify"] + actions = [o_feed_force_update, o_feed_export_as_opml, o_feed_batch_modify] + list_per_page = 20 def save_formset(self, request, form, formset, change): instances = formset.save(commit=False) @@ -333,11 +46,11 @@ def save_formset(self, request, form, formset, change): if instance.o_feed.pk: # 不保存o_feed为空的T_Feed实例 instance.status = None instance.save() - self.revoke_tasks_by_arg(instance.sid) - update_translated_feed(instance.sid, force=True) + #revoke_tasks_by_arg(instance.sid) + update_translated_feed.schedule(args=(instance.sid,True), delay=1) for instance in formset.deleted_objects: - self.revoke_tasks_by_arg(instance.sid) + #revoke_tasks_by_arg(instance.sid) instance.delete() formset.save_m2m() @@ -350,51 +63,43 @@ def save_model(self, request, obj, form, change): # translator_changed = 'content_type' in form.changed_data or 'object_id' in form.changed_data if feed_url_changed or translation_display_changed: obj.valid = None - obj.name = "Loading" if not obj.name else obj.name + obj.name = obj.name or "Loading" obj.save() update_original_feed.schedule( - args=(obj.sid,), delay=1 + args=(obj.sid,True), delay=1 ) # 会执行一次save() # 不放在model的save里是为了排除translator的更新，省流量 elif frequency_changed: obj.save() - self.revoke_tasks_by_arg(obj.sid) + #revoke_tasks_by_arg(obj.sid) update_original_feed.schedule( args=(obj.sid,), delay=obj.update_frequency * 60 ) else: - obj.name = "Empty" if not obj.name else obj.name + obj.name = obj.name or "Empty" obj.save() + @admin.display(description=_("Translator")) def translator(self, obj): return obj.translator - translator.short_description = _("Translator") - + @admin.display(description=_("Translated Language")) def translated_language(self, obj): return ", ".join(t_feed.language for t_feed in obj.t_feed_set.all()) - translated_language.short_description = _("Translated Language") - # def get_queryset(self, request): # return super().get_queryset(request).prefetch_related('tags') - # def tag_list(self, obj): # return ", ".join(o.name for o in obj.tags.all()) + @admin.display(description=_("Size(KB)"), ordering="size") def size_in_kb(self, obj): return int(obj.size / 1024) - size_in_kb.short_description = _("Size(KB)") - + @admin.display(description=_("Valid"), ordering="valid") def is_valid(self, obj): return valid_icon(obj.valid) - is_valid.short_description = _("Valid") - - is_valid.admin_order_field = "valid" - size_in_kb.admin_order_field = "size" - - @admin.display(description="feed_url") + @admin.display(description=_("Feed URL")) def show_feed_url(self, obj): if obj.feed_url: url = obj.feed_url @@ -406,8 +111,6 @@ def show_feed_url(self, obj): ) return "" - show_feed_url.short_description = _("Feed URL") - def proxy_feed_url(self, obj): if obj.sid: return format_html( @@ -415,96 +118,8 @@ def proxy_feed_url(self, obj): ) return "" - def o_feed_batch_modify(self, request, queryset): - if "apply" in request.POST: - logging.info("Apply o_feed_batch_modify") - post_data = request.POST - fields = { - "update_frequency": "update_frequency_value", - "max_posts": "max_posts_value", - "translator": "translator_value", - "translation_display": "translation_display_value", - "summary_engine": "summary_engine_value", - "summary_detail": "summary_detail_value", - "additional_prompt": "additional_prompt_value", - "fetch_article": "fetch_article", - "quality": "quality", - "category": "category_value", - } - field_types = { - "update_frequency": int, - "max_posts": int, - "translation_display": int, - "summary_detail": float, - "additional_prompt": str, - "fetch_article": literal_eval, - "quality": literal_eval, - } - update_fields = {} - # tags_value = None - - for field, value_field in fields.items(): - value = post_data.get(value_field) - if post_data.get(field, "Keep") != "Keep" and value: - match field: - case "translator": - content_type_id, object_id = map(int, value.split(":")) - update_fields["content_type_id"] = content_type_id - update_fields["object_id"] = object_id - case "summary_engine": - content_type_summary_id, object_id_summary = map( - int, value.split(":") - ) - update_fields["content_type_summary_id"] = ( - content_type_summary_id - ) - update_fields["object_id_summary"] = object_id_summary - case "category": - tag_model = O_Feed.category.tag_model - category_o, _ = tag_model.objects.get_or_create(name=value) - update_fields["category"] = category_o - case _: - update_fields[field] = field_types.get(field, str)(value) - - if update_fields: - queryset.update(**update_fields) - # for obj in queryset: - # obj.category.update_count() - - # if tags_value is not None: - # for obj in queryset: - # obj.tags = [*tags_value] - # obj.save() - # O_Feed.objects.bulk_update(queryset, ['tags'])?? - - # self.message_user(request, f"Successfully modified {queryset.count()} items.") - # return HttpResponseRedirect(request.get_full_path()) - return redirect(request.get_full_path()) - - translator_choices, summary_engine_choices = ( - get_translator_and_summary_choices() - ) - logging.info( - "translator_choices: %s, summary_engine_choices: %s", - translator_choices, - summary_engine_choices, - ) - return render( - request, - "admin/o_feed_batch_modify.html", - context={ - **core_admin_site.each_context(request), - "items": queryset, - "translator_choices": translator_choices, - "summary_engine_choices": summary_engine_choices, - }, - ) - - o_feed_batch_modify.short_description = _("Batch modification") - - -class T_FeedAdmin(admin.ModelAdmin, CustomModelActions): +class T_FeedAdmin(admin.ModelAdmin): list_display = [ "id", "feed_url", @@ -531,7 +146,8 @@ class T_FeedAdmin(admin.ModelAdmin, CustomModelActions): "size", "modified", ] - actions = ["t_feed_force_update", "t_feed_export_as_opml", "t_feed_batch_modify"] + actions = [t_feed_force_update, t_feed_export_as_opml, t_feed_batch_modify] + list_per_page = 20 # def get_search_results(self, request, queryset, search_term): # queryset, use_distinct = super().get_search_results(request, queryset, search_term) # queryset |= self.model.objects.filter(o_feed__feed_url__icontains=search_term) @@ -561,47 +177,6 @@ def status_icon(self, obj): status_icon.short_description = _("Status") status_icon.admin_order_field = "status" - def t_feed_batch_modify(self, request, queryset): - if "apply" in request.POST: - logging.info("Apply t_feed_batch_modify") - translate_title = request.POST.get("translate_title", "Keep") - translate_content = request.POST.get("translate_content", "Keep") - summary = request.POST.get("summary", "Keep") - match translate_title: - case "Keep": - pass - case "True": - queryset.update(translate_title=True) - case "False": - queryset.update(translate_title=False) - - match translate_content: - case "Keep": - pass - case "True": - queryset.update(translate_content=True) - case "False": - queryset.update(translate_content=False) - - match summary: - case "Keep": - pass - case "True": - queryset.update(summary=True) - case "False": - queryset.update(summary=False) - - # self.message_user(request, f"Successfully modified {queryset.count()} items.") - # return HttpResponseRedirect(request.get_full_path()) - return redirect(request.get_full_path()) - return render( - request, - "admin/t_feed_batch_modify.html", - context={**core_admin_site.each_context(request), "items": queryset}, - ) - - t_feed_batch_modify.short_description = _("Batch modification") - core_admin_site.register(O_Feed, O_FeedAdmin) core_admin_site.register(T_Feed, T_FeedAdmin) diff --git a/core/custom_admin_site.py b/core/custom_admin_site.py new file mode 100644 index 0000000..a97ba4d --- /dev/null +++ b/core/custom_admin_site.py @@ -0,0 +1,133 @@ +from django.contrib.admin import AdminSite +from django.urls import path +from django.utils.translation import gettext_lazy as _ +from django.utils.http import url_has_allowed_host_and_scheme +from django.core.paginator import Paginator +from django.shortcuts import render, redirect + +from utils.modelAdmin_utils import ( + get_all_app_models, + valid_icon, +) +class CoreAdminSite(AdminSite): + site_header = _("RSS Translator Admin") + site_title = _("RSS Translator") + index_title = _("Dashboard") + + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path("translator/add", translator_add_view, name="translator_add"), + path("translator/list", translator_list_view, name="translator_list"), + ] + return custom_urls + urls + + def get_app_list(self, request, app_label=None): + app_list = super().get_app_list(request, app_label) + app_list += [ + { + "name": _("Engine"), + "app_label": "engine", + "models": [ + { + "name": _("Translator"), + "object_name": "Translator", + "admin_url": "/translator/list", + "add_url": "/translator/add", + # "view_only": False, + } + ], + } + ] + + return app_list + + +class TranslatorPaginator(Paginator): + def __init__(self): + super().__init__(self, 100) + + self.translator_count = len(get_all_app_models("translator")) + + @property + def count(self): + return self.translator_count + + def page(self, number): + limit = self.per_page + offset = (number - 1) * self.per_page + return self._get_page( + self.enqueued_items(limit, offset), + number, + self, + ) + + # Copied from Huey's SqliteStorage with some modifications to allow pagination + def enqueued_items(self, limit, offset): + translators = get_all_app_models("translator") + translator_list = [] + for model in translators: + objects = ( + model.objects.all() + .order_by("name") + .values_list("id", "name", "valid")[offset : offset + limit] + ) + for obj_id, obj_name, obj_valid in objects: + translator_list.append( + { + "id": obj_id, + "table_name": model._meta.db_table.split("_")[1], + "name": obj_name, + "valid": valid_icon(obj_valid), + "provider": model._meta.verbose_name, + } + ) + + return translator_list + + +def translator_list_view(request): + page_number = int(request.GET.get("p", 1)) + paginator = TranslatorPaginator() + page = paginator.get_page(page_number) + page_range = paginator.get_elided_page_range(page_number, on_each_side=2, on_ends=2) + + context = { + **core_admin_site.each_context(request), + "title": "Translator", + "page": page, + "page_range": page_range, + "translators": page.object_list, + } + return render(request, "admin/translator.html", context) + + +def translator_add_view(request): + if request.method == "POST": + translator_name = request.POST.get("translator_name", "/") + # redirect to example.com/translator/translator_name/add + target = f"/translator/{translator_name}/add" + return ( + redirect(target) + if url_has_allowed_host_and_scheme(target, allowed_hosts=None) + else redirect("/") + ) + else: + models = get_all_app_models("translator") + translator_list = [] + for model in models: + translator_list.append( + { + "table_name": model._meta.db_table.split("_")[1], + "provider": model._meta.verbose_name, + } + ) + context = { + **core_admin_site.each_context(request), + "translator_choices": translator_list, + } + return render(request, "admin/translator_add.html", context) + + +core_admin_site = CoreAdminSite() + diff --git a/core/forms.py b/core/forms.py new file mode 100644 index 0000000..860f5da --- /dev/null +++ b/core/forms.py @@ -0,0 +1,97 @@ +from django import forms +from django.db import transaction +from django.utils.translation import gettext_lazy as _ +from .models import O_Feed, T_Feed +from utils.modelAdmin_utils import get_translator_and_summary_choices + +class O_FeedForm(forms.ModelForm): + # 自定义字段，使用ChoiceField生成下拉菜单 + translator = forms.ChoiceField( + choices=(), + required=False, + help_text=_("Select a valid translator"), + label=_("Translator"), + ) + summary_engine = forms.ChoiceField( + choices=(), + required=False, + help_text=_("Select a valid AI engine"), + label=_("Summary Engine"), + ) + + def __init__(self, *args, **kwargs): + super(O_FeedForm, self).__init__(*args, **kwargs) + self.fields["translator"].choices, self.fields["summary_engine"].choices = ( + get_translator_and_summary_choices() + ) + + # 如果已经有关联的对象，设置默认值 + instance = getattr(self, "instance", None) + if instance and instance.pk: + self._set_initial_values(instance) + + def _set_initial_values(self, instance): + if instance.content_type and instance.object_id: + self.fields["translator"].initial = f"{instance.content_type.id}:{instance.object_id}" + if instance.content_type_summary and instance.object_id_summary: + self.fields["summary_engine"].initial = f"{instance.content_type_summary.id}:{instance.object_id_summary}" + + class Meta: + model = O_Feed + fields = [ + "feed_url", + "update_frequency", + "max_posts", + "translator", + "translation_display", + "summary_engine", + "summary_detail", + "additional_prompt", + "fetch_article", + "quality", + "name", + "category", + ] + + def _process_translator(self, instance): + if self.cleaned_data["translator"]: + content_type_id, object_id = map(int, self.cleaned_data["translator"].split(":")) + instance.content_type_id = content_type_id + instance.object_id = object_id + else: + instance.content_type = None + instance.object_id = None + + def _process_summary_engine(self, instance): + if self.cleaned_data["summary_engine"]: + content_type_summary_id, object_id_summary = map(int, self.cleaned_data["summary_engine"].split(":")) + instance.content_type_summary_id = content_type_summary_id + instance.object_id_summary = object_id_summary + else: + instance.content_type_summary_id = None + instance.object_id_summary = None + + # 重写save方法，以处理自定义字段的数据 + @transaction.atomic + def save(self, commit=True): + instance = super(O_FeedForm, self).save(commit=False) + + self._process_translator(instance) + self._process_summary_engine(instance) + + if commit: + instance.save() + + return instance + +class T_FeedForm(forms.ModelForm): + class Meta: + model = T_Feed + fields = ["language", "translate_title", "sid"] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["sid"].required = False + if self.instance.pk: + self.fields["language"].disabled = True + self.fields["sid"].disabled = True \ No newline at end of file diff --git a/core/inlines.py b/core/inlines.py new file mode 100644 index 0000000..d0b94cd --- /dev/null +++ b/core/inlines.py @@ -0,0 +1,69 @@ + +from django.contrib import admin +from django.urls import reverse +from django.utils.html import format_html +from django.utils.translation import gettext_lazy as _ +from .models import T_Feed +from .forms import T_FeedForm +from utils.modelAdmin_utils import ( + valid_icon, +) + +class T_FeedInline(admin.TabularInline): + model = T_Feed + form = T_FeedForm + fields = [ + "language", + "obj_status", + "feed_url", + "translate_title", + "translate_content", + "summary", + "total_tokens", + "total_characters", + "size_in_kb", + "sid", + ] + readonly_fields = ( + "feed_url", + "obj_status", + "size_in_kb", + "total_tokens", + "total_characters", + ) + extra = 1 + + @admin.display(description=_("Translated Feed URL")) + def feed_url(self, obj): + if obj.sid: + rss = reverse("core:rss", kwargs={"feed_sid": obj.sid}) + rss_url = self.request.build_absolute_uri(rss) + json = reverse("core:json", kwargs={"feed_sid": obj.sid}) + json_url = self.request.build_absolute_uri(json) + return format_html( + "RSS " + " | " + "JSON " + "", + rss_url, + json_url, + ) + return "" + + class Media: + js = ("js/admin/copytoclipboard.js",) + + @admin.display(description=_("Size(KB)")) + def size_in_kb(self, obj): + return int(obj.size / 1024) + + @admin.display(description=_("Status")) + def obj_status(self, obj): + if not obj.pk: + return "" + return valid_icon(obj.status) + + def get_formset(self, request, obj=None, **kwargs): + # Store the request for use in feed_url + self.request = request + return super(T_FeedInline, self).get_formset(request, obj, **kwargs) diff --git a/core/tasks.py b/core/tasks.py index 2a02d96..248a52f 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -11,7 +11,7 @@ from django.db import IntegrityError from huey.contrib.djhuey import HUEY as huey -from huey.contrib.djhuey import on_startup, db_task +from huey.contrib.djhuey import on_startup, db_task, on_shutdown from .models import O_Feed, T_Feed from translator.models import TranslatorEngine, Translated_Content @@ -28,6 +28,12 @@ # from huey_monitor.models import TaskModel unique_tasks = set() +def revoke_tasks_by_arg(arg_to_match): + for task in huey.scheduled() + huey.pending(): + # Assuming the first argument is the one we're interested in (e.g., obj.pk) + if task.args and task.args[0] == arg_to_match: + logging.info("Revoke task: %s", task) + huey.revoke_by_id(task) # @periodic_task(crontab( minute='*/1')) @on_startup() @@ -42,19 +48,15 @@ def schedule_update(): args=(feed.sid,), delay=feed.update_frequency * 60 ) - -# @on_shutdown() -# def flush_all(): -# huey.storage.flush_queue() -# huey.storage.flush_schedule() -# huey.storage.flush_results() -# clean TaskModel all data -# TaskModel.objects.all().delete() +@on_shutdown() +def cleanup_tasks(): + huey.storage.flush_all() @db_task(retries=3) -def update_original_feed(sid: str): - if sid in unique_tasks: +def update_original_feed(sid: str, force:bool = False): + if sid in unique_tasks: # 如果判断force的话，是没法停止正在执行的task + logging.warning("(skip)This task update_original_feed is executing: %s",sid) return else: unique_tasks.add(sid) @@ -65,7 +67,9 @@ def update_original_feed(sid: str): except O_Feed.DoesNotExist: return False + revoke_tasks_by_arg(sid) logging.info("Call task update_original_feed: %s", obj.feed_url) + feed_dir_path = Path(settings.DATA_FOLDER) / "feeds" if not os.path.exists(feed_dir_path): @@ -119,8 +123,9 @@ def update_original_feed(sid: str): @db_task(retries=3) -def update_translated_feed(sid: str, force=False): - if sid in unique_tasks and not force: +def update_translated_feed(sid: str, force:bool = False): + if sid in unique_tasks: # 如果判断force的话，是没法停止正在执行的task + logging.warning("(skip)The task update_translated_feed is executing: %s",sid) return else: unique_tasks.add(sid) @@ -133,6 +138,7 @@ def update_translated_feed(sid: str, force=False): return False try: + revoke_tasks_by_arg(sid) logging.info("Call task update_translated_feed: %s", obj.o_feed.feed_url) if obj.o_feed.pk is None: @@ -157,9 +163,6 @@ def update_translated_feed(sid: str, force=False): return False translated_feed_file_path = f"{feed_dir_path}/{obj.sid}" - # if not os.path.exists(translated_feed_file_path): - # with open(translated_feed_file_path, "w", encoding="utf-8") as f: - # f.write("Translation in progress...") original_feed = feedparser.parse(original_feed_file_path) diff --git a/core/urls.py b/core/urls.py index 280944c..84caad9 100644 --- a/core/urls.py +++ b/core/urls.py @@ -10,6 +10,7 @@ path("all//", views.all, name="all"), path("category/", views.category, name="category"), path("category//", views.category, name="category"), + path("json/", views.rss_json, name="json"), path("json//", views.rss_json, name="json"), re_path(r"(?P[^/]+)/?$", views.rss, name="rss"), ] diff --git a/dev-diaries/versun-dev-diary.md b/dev-diaries/versun-dev-diary.md new file mode 100644 index 0000000..92cee17 --- /dev/null +++ b/dev-diaries/versun-dev-diary.md @@ -0,0 +1,24 @@ +# 2024-07-04 +想使用[django-tasks](https://github.com/RealOrangeOne/django-tasks)来代替huey做为任务管理，但因为还在开发中，所以担心稳定性。主要是因为它的issues中有几个我在使用huey时遇到的问题，且正在解决，但huey还遥遥无期。。。。 + +添加了huey的shutdown动作，flush所有任务（包括revoke的任务，防止无用的任务堆积） + +huey在revoke一个task后，并不会删除这个task，如果之后继续revoke的话，还会重复revoke一遍，本来想搭配restore_by_id来恢复并重新预约任务，但需要存储task的result才能调用reschedule，所以暂时放弃了。 + +先这样吧，又不是不能用，等django-tasks完善吧 + +# 2024-07-03 +看来还是有必要添加个开发日记，因为有些代码现在看起来很傻逼，但就是不知道当初为啥会这么写，感觉是为了某种边缘情况，但就是记不起来。 + +开发日记的灵感来源于：https://github.com/cozemble/breezbook/discussions/32 + +--- +今天主要是重构了core/admin.py，原先的代码太乱了。 +整个文件直接丢给pplx的claude 3.5 sonnet，效果非常好，基本一次就成了 + +然后顺便发现了revoke_tasks_by_arg函数的使用逻辑有点问题: +t_feed_force_update不应该调用revoke_tasks_by_arg，因为它是通过update_original_feed task来调用的，只有original_feed才会预约，而update_translated_feed task是实时性的，所以应该在update_translated_feed里检查任务唯一性就行了。 + +o_feed_force_update也不用revoke_tasks_by_arg，直接放在update_original_feed task更方便直观，否则task完全不知道可能会在哪里被revoke掉 + +新发布的版本，还是先发布pre release，push到docker dev标签，自己先测试一段时间再push到latest \ No newline at end of file diff --git a/templates/admin/base_site.html b/templates/admin/base_site.html index fec1a50..c6ee290 100644 --- a/templates/admin/base_site.html +++ b/templates/admin/base_site.html @@ -1,47 +1,77 @@ {% extends "admin/base_site.html" %} +{% block extrahead %} +{{ block.super }} + + +{% endblock %} + {% block footer %} -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/translator/models/base.py b/translator/models/base.py index 87af615..deacf7f 100644 --- a/translator/models/base.py +++ b/translator/models/base.py @@ -181,7 +181,7 @@ def translate( # else: # translated_text = '' # logging.warning("Translator->%s: %s", res.choices[0].finish_reason, text) - tokens = res.usage.total_tokens + tokens = res.usage.total_tokens if res.usage else 0 except Exception as e: logging.error("OpenAIInterface->%s: %s", e, text) diff --git a/translator/models/deeplweb.py b/translator/models/deeplweb.py index 308a7b8..5f68de0 100644 --- a/translator/models/deeplweb.py +++ b/translator/models/deeplweb.py @@ -39,13 +39,13 @@ class Meta: def validate(self) -> bool: try: - resp = self.translate("Hello World", "Chinese Simplified") + resp = self.translate("Hello World", "Chinese Simplified", validate=True) return resp.get("text") != "" except Exception as e: logging.error("DeepLWebTranslator validate ->%s", e) return False - def translate(self, text: str, target_language: str, **kwargs) -> dict: + def translate(self, text: str, target_language: str, validate:bool=False, **kwargs) -> dict: logging.info(">>> DeepL Web Translate [%s]: %s", target_language, text) target_code = self.language_code_map.get(target_language, None) translated_text = "" @@ -62,5 +62,6 @@ def translate(self, text: str, target_language: str, **kwargs) -> dict: except Exception as e: logging.error("DeepLWebTranslator->%s: %s", e, text) finally: - sleep(self.interval) + if not validate: + sleep(self.interval) return {"text": translated_text, "characters": len(text)} diff --git a/translator/models/deeplx.py b/translator/models/deeplx.py index ff43fc3..35387e4 100644 --- a/translator/models/deeplx.py +++ b/translator/models/deeplx.py @@ -42,12 +42,12 @@ class Meta: def validate(self) -> bool: try: - resp = self.translate("Hello World", "Chinese Simplified") + resp = self.translate("Hello World", "Chinese Simplified", validate=True) return resp.get("text") != "" except Exception: return False - def translate(self, text: str, target_language: str, **kwargs) -> dict: + def translate(self, text: str, target_language: str, validate:bool=False, **kwargs) -> dict: logging.info(">>> DeepLX Translate [%s]: %s", target_language, text) target_code = self.language_code_map.get(target_language, None) translated_text = "" @@ -73,5 +73,6 @@ def translate(self, text: str, target_language: str, **kwargs) -> dict: except Exception as e: logging.error("DeepLXTranslator->%s: %s", e, text) finally: - sleep(self.interval) + if not validate: + sleep(self.interval) return {"text": translated_text, "characters": len(text)} diff --git a/translator/models/doubao.py b/translator/models/doubao.py index bccafc5..dedc014 100644 --- a/translator/models/doubao.py +++ b/translator/models/doubao.py @@ -72,7 +72,8 @@ def translate(self, text: str, target_language: str, system_prompt: str = None, if res.choices and res.choices[0].message.content: translated_text = res.choices[0].message.content logging.info("DoubaoTranslator->%s: %s", res.choices[0].finish_reason, translated_text) - tokens = res.usage.total_tokens + + tokens = res.usage.total_tokens if res.usage else 0 except Exception as e: logging.error("DoubaoTranslator->%s: %s", e, text) diff --git a/translator/models/google_translate_web.py b/translator/models/google_translate_web.py index d6227bb..8595635 100644 --- a/translator/models/google_translate_web.py +++ b/translator/models/google_translate_web.py @@ -41,10 +41,10 @@ class Meta: verbose_name_plural = "Google Translate(Web)" def validate(self) -> bool: - results = self.translate("hi", "Chinese Simplified") + results = self.translate("hi", "Chinese Simplified", validate=True) return results.get("text") != "" - def translate(self, text: str, target_language: str, **kwargs) -> dict: + def translate(self, text: str, target_language: str, validate:bool=False, **kwargs) -> dict: logging.info(">>> Google Translate Web Translate [%s]:", target_language) target_language = self.language_code_map.get(target_language) translated_text = "" @@ -71,5 +71,6 @@ def translate(self, text: str, target_language: str, **kwargs) -> dict: except Exception as e: logging.error("GoogleTranslateWebTranslator->%s: %s", e, text) finally: - sleep(self.interval) + if not validate: + sleep(self.interval) return {"text": translated_text, "characters": len(text)} diff --git a/utils/modelAdmin_utils.py b/utils/modelAdmin_utils.py index 1cdd0c0..196f4f2 100644 --- a/utils/modelAdmin_utils.py +++ b/utils/modelAdmin_utils.py @@ -1,150 +1,12 @@ -import logging -from datetime import datetime from django.conf import settings -from django.urls import reverse -from django.http import HttpResponse from django.utils.html import format_html from django.apps import apps - - -from opyml import OPML, Outline, Head -from huey.contrib.djhuey import HUEY as huey - -# from django.conf import settings from django.utils.translation import gettext_lazy as _ -from django.db import transaction from django.contrib.contenttypes.models import ContentType - -from core.tasks import update_original_feed, update_translated_feed - # if settings.DEBUG: # from huey_monitor.models import TaskModel - -class CustomModelActions: - def o_feed_export_as_opml(self, request, queryset): - try: - opml_obj = OPML() - opml_obj.head = Head( - title="Original Feeds | RSS Translator", - date_created=datetime.now().strftime("%a, %d %b %Y %H:%M:%S %z"), - owner_name="RSS Translator", - ) - - categories = {} - for item in queryset: - category = item.category.name if item.category else 'default' - #category_outline = Outline(text=category) - if category not in categories: - categories[category] = Outline(text=category) - - item_outline = Outline( - title=item.name, - text=item.name, - type="rss", - xml_url=item.feed_url, - html_url=item.feed_url, - ) - categories[category].outlines.append(item_outline) - - #category_outline.outlines.append(item_outline) - #opml_obj.body.outlines.append(category_outline) - for category_outline in categories.values(): - opml_obj.body.outlines.append(category_outline) - - response = HttpResponse(opml_obj.to_xml(), content_type="application/xml") - response["Content-Disposition"] = ( - 'attachment; filename="rsstranslator_original_feeds.opml"' - ) - return response - except Exception as e: - logging.error("o_feed_export_as_opml: %s", str(e)) - return HttpResponse("An error occurred", status=500) - - o_feed_export_as_opml.short_description = _("Export selected feeds as OPML") - - def t_feed_export_as_opml(self, request, queryset): - try: - opml_obj = OPML() - opml_obj.head = Head( - title="Translated Feeds | RSS Translator", - date_created=datetime.now().strftime("%a, %d %b %Y %H:%M:%S %z"), - owner_name="RSS Translator", - ) - - categories = {} - for item in queryset: - category = item.o_feed.category.name if item.o_feed.category else 'default' - text = item.o_feed.name or "No Name" - xml_url = request.build_absolute_uri( - reverse("core:rss", kwargs={"feed_sid": item.sid}) - ) - - if category not in categories: - categories[category] = Outline(text=category) - - item_outline = Outline( - title=text, - text=text, - type="rss", - xml_url=xml_url, - html_url=item.o_feed.feed_url, - ) - categories[category].outlines.append(item_outline) - for category_outline in categories.values(): - opml_obj.body.outlines.append(category_outline) - - response = HttpResponse(opml_obj.to_xml(), content_type="application/xml") - response["Content-Disposition"] = ( - 'attachment; filename="rsstranslator_translated_feeds.opml"' - ) - return response - except Exception as e: - logging.error("t_feed_export_as_opml: %s", str(e)) - return HttpResponse("An error occurred", status=500) - - t_feed_export_as_opml.short_description = _("Export selected feeds as OPML") - - def o_feed_force_update(self, request, queryset): - logging.info("Call o_feed_force_update: %s", queryset) - with transaction.atomic(): - for instance in queryset: - instance.etag = "" - instance.valid = None - instance.save() - self.revoke_tasks_by_arg(instance.sid) - update_original_feed.schedule( - args=(instance.sid,), delay=1 - ) # 会执行一次save() - - o_feed_force_update.short_description = _("Force update") - - def t_feed_force_update(self, request, queryset): - logging.info("Call t_feed_force_update: %s", queryset) - with transaction.atomic(): - for instance in queryset: - instance.modified = None - instance.status = None - instance.save() - self.revoke_tasks_by_arg(instance.sid) - update_translated_feed.schedule( - args=(instance.sid,), delay=1 - ) # 会执行一次save() - - t_feed_force_update.short_description = _("Force update") - - def revoke_tasks_by_arg(self, arg_to_match): - for task in huey.scheduled(): - # Assuming the first argument is the one we're interested in (e.g., obj.pk) - if task.args and task.args[0] == arg_to_match: - logging.info("Revoke task: %s", task) - huey.revoke_by_id(task) - # delete TaskModel data - # if settings.DEBUG: - # TaskModel.objects.filter(task_id=task.id).delete() - - # def get_all_subclasses(cls): # subclasses = set() # for subclass in cls.__subclasses__(): diff --git a/utils/text_handler.py b/utils/text_handler.py index fec8d85..64f1fbf 100644 --- a/utils/text_handler.py +++ b/utils/text_handler.py @@ -137,6 +137,7 @@ def should_skip(element): "bdo", "cite", "dfn", + "iframe", ] if isinstance(element, Comment): return True