diff --git a/core/utils/bot.py b/core/utils/bot.py index b80a4f83..cd3488fc 100644 --- a/core/utils/bot.py +++ b/core/utils/bot.py @@ -13,6 +13,7 @@ from core.queue import JobQueue from core.scheduler import Scheduler from core.types import PrivateAssets, Secret from core.utils.info import Info +from core.utils.web_render import check_web_render async def init_async(start_scheduler=True) -> None: @@ -33,9 +34,10 @@ async def init_async(start_scheduler=True) -> None: Scheduler.start() logging.getLogger('apscheduler.executors.default').setLevel(logging.WARNING) await load_secret() + await check_web_render() try: Info.version = os.popen('git rev-parse HEAD', 'r').read() - except Exception as e: + except Exception: Logger.warn(f'Failed to get Git commit hash, is it a Git repository?') Logger.info(f'Hello, {bot_name}!') diff --git a/core/utils/image_table.py b/core/utils/image_table.py index b5ff1ff7..da721c9b 100644 --- a/core/utils/image_table.py +++ b/core/utils/image_table.py @@ -6,13 +6,10 @@ import aiohttp import ujson as json from tabulate import tabulate -from config import CFG from core.logger import Logger from .cache import random_cache_path from .http import download_to_cache - -web_render = CFG.get_url('web_render') -web_render_local = CFG.get_url('web_render_local') +from .web_render import WebRender, webrender class ImageTable: @@ -22,10 +19,7 @@ class ImageTable: async def image_table_render(table: Union[ImageTable, List[ImageTable]], save_source=True, use_local=True): - if not web_render_local: - if not web_render: - Logger.warn('[Webrender] Webrender is not configured.') - return False + if not WebRender.local use_local = False pic = False @@ -67,7 +61,7 @@ async def image_table_render(table: Union[ImageTable, List[ImageTable]], save_so try: pic = await download_to_cache( - web_render_local if use_local else web_render, + webrender(), method='POST', post_data=json.dumps(html), request_private_ip=True, @@ -78,7 +72,7 @@ async def image_table_render(table: Union[ImageTable, List[ImageTable]], save_so except aiohttp.ClientConnectorError: if use_local: pic = await download_to_cache( - web_render, + webrender(use_local=False), method='POST', post_data=json.dumps(html), request_private_ip=True, diff --git a/core/utils/web_render.py b/core/utils/web_render.py new file mode 100644 index 00000000..4705635f --- /dev/null +++ b/core/utils/web_render.py @@ -0,0 +1,59 @@ +import traceback + +from config import CFG +from core.logger import Logger +from core.utils.http import get_url +from core.utils.ip import IP + +web_render = CFG.get_url('web_render') +web_render_local = CFG.get_url('web_render_local') + +class WebRender: + status = False + local = False + + +def webrender(method: str = None, url: str = None, use_local: bool = True): + '''根据请求方法生成 Webrender URL。 + + :param method: API 方法。 + :param url: 若 method 为 source,则指定请求的 URL。 + :param use_local: 是否使用本地 Webrender。 + :returns: 生成的 Webrender URL。 + ''' + if use_local and not WebRender.local: + use_local = False + if method == 'source': + if WebRender.status: + return f'{(web_render_local if use_local else web_render)}source?url={url}' + else: + return url + else: + if WebRender.status: + return (web_render_local if use_local else web_render) + method + else: + return None + + +async def check_web_render(): + if not web_render_local: + if not web_render: + Logger.warn('[Webrender] Webrender is not configured.') + else: + WebRender.status = True + else: + WebRender.local = True + WebRender.status = True + if IP.country != 'China': + ping_url = 'http://www.google.com/' + else: + ping_url = 'http://www.baidu.com/' + if WebRender.status: + try: + Logger.info('[Webrender] Checking Webrender status...') + await get_url(webrender('source', ping_url), 200, request_private_ip=True) + Logger.info('[Webrender] Webrender is working as expected.') + except Exception: + Logger.error('[Webrender] Webrender is not working as expected.') + Logger.error(traceback.format_exc()) + WebRender.status = False \ No newline at end of file diff --git a/modules/arcaea/__init__.py b/modules/arcaea/__init__.py index 9eba70dd..0446daaa 100644 --- a/modules/arcaea/__init__.py +++ b/modules/arcaea/__init__.py @@ -6,6 +6,7 @@ from core.builtins import Bot, Image as BImage, Plain from core.component import module from core.utils.cache import random_cache_path from core.utils.http import get_url +from core.utils.web_render import webrender assets_path = os.path.abspath('./assets/arcaea') @@ -17,8 +18,8 @@ arc = module('arcaea', developers=['OasisAkari'], desc='{arcaea.help.desc}', @arc.command('download {{arcaea.help.download}}') async def _(msg: Bot.MessageSession): - resp = await get_url('https://webapi.lowiro.com/webapi/serve/static/bin/arcaea/apk/', 200, - fmt='json') + url = 'https://webapi.lowiro.com/webapi/serve/static/bin/arcaea/apk/' + resp = await get_url(webrender('source', url), 200, fmt='json', request_private_ip=True) if resp: await msg.finish([Plain(msg.locale.t("arcaea.message.download", version=resp["value"]["version"], url=resp['value']['url']))]) @@ -28,8 +29,8 @@ async def _(msg: Bot.MessageSession): @arc.command('random {{arcaea.help.random}}') async def _(msg: Bot.MessageSession): - resp = await get_url('https://webapi.lowiro.com/webapi/song/showcase/', - 200, fmt='json') + url = 'https://webapi.lowiro.com/webapi/song/showcase/' + resp = await get_url(webrender('source', url), 200, fmt='json', request_private_ip=True) if resp: value = resp["value"][0] image = f'{assets_path}/jacket/{value["song_id"]}.jpg' @@ -45,11 +46,11 @@ async def _(msg: Bot.MessageSession): 'rank paid {{arcaea.help.rank.paid}}') async def _(msg: Bot.MessageSession): if msg.parsed_msg.get('free', False): - resp = await get_url('https://webapi.lowiro.com/webapi/song/rank/free/', - 200, fmt='json') + url = 'https://webapi.lowiro.com/webapi/song/rank/free/' + resp = await get_url(webrender('source', url), 200, fmt='json', request_private_ip=True) else: - resp = await get_url('https://webapi.lowiro.com/webapi/song/rank/paid/', - 200, fmt='json') + url = 'https://webapi.lowiro.com/webapi/song/rank/paid/' + resp = await get_url(webrender('source', url), 200, fmt='json', request_private_ip=True) if resp: r = [] rank = 0 diff --git a/modules/bugtracker/bugtracker.py b/modules/bugtracker/bugtracker.py index bcfe597f..de3df51c 100644 --- a/modules/bugtracker/bugtracker.py +++ b/modules/bugtracker/bugtracker.py @@ -5,13 +5,11 @@ import aiohttp import ujson as json from PIL import ImageFont -from config import CFG from core.builtins import Url from core.logger import Logger from core.utils.http import download_to_cache, get_url +from core.utils.web_render import WebRender, webrender -web_render = CFG.get_url('web_render') -web_render_local = CFG.get_url('web_render_local') elements = ['div#descriptionmodule'] assets_path = os.path.abspath('./assets/') @@ -20,14 +18,11 @@ spx_cache = {} async def make_screenshot(page_link, use_local=True): elements_ = elements.copy() - if not web_render_local: - if not web_render: - Logger.warn('[Webrender] Webrender is not configured.') - return False + if not WebRender.local use_local = False Logger.info('[Webrender] Generating element screenshot...') try: - img = await download_to_cache((web_render_local if use_local else web_render) + 'element_screenshot', + img = await download_to_cache(webrender('element_screenshot', use_local=use_local), status_code=200, headers={'Content-Type': 'application/json'}, method="POST", diff --git a/modules/core/console.py b/modules/core/console.py deleted file mode 100644 index 19e9379f..00000000 --- a/modules/core/console.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys - -from core.builtins import Bot -from core.component import module - -exit = module('exit', base=True, available_for=['TEST|Console']) - - -@exit.command() -async def _(msg: Bot.MessageSession): - confirm = await msg.wait_confirm(msg.locale.t("core.message.confirm"), append_instruction=False, delete=False) - if confirm: - print('Exited.') - sys.exit() \ No newline at end of file diff --git a/modules/core/su_utils.py b/modules/core/su_utils.py index 4541ed65..7674e047 100644 --- a/modules/core/su_utils.py +++ b/modules/core/su_utils.py @@ -295,6 +295,16 @@ if Info.subprocess: await msg.finish() +exit_ = module('exit', required_superuser=True, base=True, available_for=['TEST|Console']) + +@exit_.command() +async def _(msg: Bot.MessageSession): + confirm = await msg.wait_confirm(msg.locale.t("core.message.confirm"), append_instruction=False, delete=False) + if confirm: + print('Exited.') + sys.exit() + + if Bot.FetchTarget.name == 'QQ': resume = module('resume', required_superuser=True) @@ -480,10 +490,10 @@ if Config('enable_petal'): msg.data.clear_petal() await msg.finish(msg.locale.t('core.message.petal.clear.self')) - lagrange = module('lagrange', required_superuser=True, base=True) +lagrange = module('lagrange', required_superuser=True, base=True) - @lagrange.command() - async def _(msg: Bot.MessageSession): - await msg.finish(f'Keepalive: {str(Temp.data.get("lagrange_keepalive", "None"))}\n' - f'Status: {str(Temp.data.get("lagrange_status", "None"))}\n' - f'Groups: {str(Temp.data.get("lagrange_available_groups", "None"))}') +@lagrange.command() +async def _(msg: Bot.MessageSession): + await msg.finish(f'Keepalive: {str(Temp.data.get("lagrange_keepalive", "None"))}\n' + f'Status: {str(Temp.data.get("lagrange_status", "None"))}\n' + f'Groups: {str(Temp.data.get("lagrange_available_groups", "None"))}') diff --git a/modules/core/utils.py b/modules/core/utils.py index f041143f..6b8afb02 100644 --- a/modules/core/utils.py +++ b/modules/core/utils.py @@ -1,22 +1,19 @@ import platform -import time -from datetime import datetime, timedelta, tzinfo +from datetime import datetime, timedelta import jwt import psutil from cpuinfo import get_cpu_info -from config import Config, CFG -from core.builtins import Bot, Embed, EmbedField, command_prefix +from config import Config +from core.builtins import Bot from core.component import module -from core.utils.http import get_url from core.utils.i18n import get_available_locales, Locale, load_locale_file from core.utils.info import Info +from core.utils.web_render import WebRender from database import BotDBUtil jwt_secret = Config('jwt_secret') -web_render = CFG.get_url('web_render') -web_render_local = CFG.get_url('web_render_local') ver = module('version', base=True) @@ -33,18 +30,6 @@ ping = module('ping', base=True) started_time = datetime.now() -async def check_web_render(use_local=True): - if not web_render_local: - if not web_render: - return False - use_local = False - try: - ping_url = 'http://www.baidu.com' - url = (web_render_local if use_local else web_render) + 'source?url=' + ping_url - await get_url(url, 200, logging_err_resp=False, request_private_ip=True) - except BaseException: - return False - return True @ping.command('{{core.help.ping}}') async def _(msg: Bot.MessageSession): @@ -59,7 +44,7 @@ async def _(msg: Bot.MessageSession): swap_percent = psutil.swap_memory().percent disk = int(psutil.disk_usage('/').used / (1024 * 1024 * 1024)) disk_total = int(psutil.disk_usage('/').total / (1024 * 1024 * 1024)) - web_render_status = str(await check_web_render()) + web_render_status = str(WebRender.status) result += '\n' + msg.locale.t("core.message.ping.detail", system_boot_time=boot_start, bot_running_time=timediff, diff --git a/modules/maimai/libraries/maimaidx_utils.py b/modules/maimai/libraries/maimaidx_utils.py index b67c1758..675a43a2 100644 --- a/modules/maimai/libraries/maimaidx_utils.py +++ b/modules/maimai/libraries/maimaidx_utils.py @@ -2,7 +2,6 @@ import os import ujson as json from datetime import datetime -from config import CFG from core.builtins import Plain from core.utils.http import get_url from core.utils.image import msgchain2image @@ -14,8 +13,6 @@ SONGS_PER_PAGE = 20 JINGLEBELL_SONG_ID = 70 assets_path = os.path.abspath('./assets/maimai') -web_render = CFG.get_url('web_render') -web_render_local = CFG.get_url('web_render_local') total_list = TotalList() plate_conversion = { diff --git a/modules/maimai/locales/zh_cn.json b/modules/maimai/locales/zh_cn.json index 3f16139f..f02629ad 100644 --- a/modules/maimai/locales/zh_cn.json +++ b/modules/maimai/locales/zh_cn.json @@ -10,6 +10,7 @@ "maimai.help.b50.beta": "查询 Maimai 用户的 B50 列表。(Beta 版)", "maimai.help.base": "根据定数或定数范围搜索歌曲。", "maimai.help.bind": "绑定 Diving-Fish 查分器用户。", + "maimai.help.calc": "根据定数计算 Rating。", "maimai.help.desc": "查询 Maimai 相关内容。", "maimai.help.grade": "查询段位认定列表。", "maimai.help.id": "根据 ID 查询歌曲或谱面信息。", @@ -29,7 +30,6 @@ "maimai.help.random": "随机一首歌曲。", "maimai.help.random.filter": "随机一首指定条件的歌曲,输入星号则表示无条件。", "maimai.help.rank": "查看用户在查分器上的分数排行。", - "maimai.help.rating": "根据定数计算 Rating。", "maimai.help.scoreline": "查询歌曲的分数线。", "maimai.help.scorelist": "查看用户在对应等级的分数列表。", "maimai.help.search": "根据歌名(或一部分)搜索歌曲。", diff --git a/modules/maimai/locales/zh_tw.json b/modules/maimai/locales/zh_tw.json index f68eac8f..1c8c3184 100644 --- a/modules/maimai/locales/zh_tw.json +++ b/modules/maimai/locales/zh_tw.json @@ -10,6 +10,7 @@ "maimai.help.b50.beta": "查詢 Maimai 使用者的 B50 列表。(Beta 版)", "maimai.help.base": "依據定數或定數範圍內搜尋歌曲。", "maimai.help.bind": "綁定 Diving-Fish 查分器使用者。", + "maimai.help.calc": "依據定數計算 Rating。", "maimai.help.desc": "查詢 Maimai 相關的內容。", "maimai.help.grade": "查詢段位認定列表。", "maimai.help.id": "依據 ID 查詢歌曲或譜面資訊。", @@ -29,7 +30,6 @@ "maimai.help.random": "隨機一首歌曲。", "maimai.help.random.filter": "隨機一首指定條件的歌曲,輸入星號則表示無條件。", "maimai.help.rank": "查看使用者在查分器上的分數排行。", - "maimai.help.rating": "依據定數計算 Rating。", "maimai.help.scoreline": "查詢歌曲的分數線。", "maimai.help.scorelist": "查看使用者在對應等級的分數列表。", "maimai.help.search": "依據歌名(或一部分)搜尋歌曲。", diff --git a/modules/maimai/maimai.py b/modules/maimai/maimai.py index 1b1df9bd..2b363860 100644 --- a/modules/maimai/maimai.py +++ b/modules/maimai/maimai.py @@ -364,7 +364,7 @@ async def _(msg: Bot.MessageSession, diff: str, sid: str, score: float): await msg.finish(msg.locale.t('maimai.message.scoreline.failed', prefix=msg.prefixes[0])) -@mai.command('rating {{maimai.help.rating}}') +@mai.command('calc {{maimai.help.calc}}') async def _(msg: Bot.MessageSession, base: float, score: float): if score: await msg.finish([Plain(max(0, computeRa(base, score)))]) diff --git a/modules/mcmod/mcmod.py b/modules/mcmod/mcmod.py index 3475d2e9..8da95631 100644 --- a/modules/mcmod/mcmod.py +++ b/modules/mcmod/mcmod.py @@ -2,26 +2,19 @@ from urllib.parse import quote from bs4 import BeautifulSoup -from config import CFG from core.builtins import Url from core.logger import Logger from core.utils.http import get_url +from core.utils.web_render import webrender api = 'https://search.mcmod.cn/s?key=' api_details = 'https://search.mcmod.cn/s?filter=3&key=' -web_render = CFG.get_url('web_render') -web_render_local = CFG.get_url('web_render_local') async def mcmod(msg, keyword: str, detail: bool = False): endpoint = api_details if detail else api search_url = endpoint + quote(keyword) - if web_render: - use_local = True if web_render_local else False - else: - return - search_url = (web_render_local if use_local else web_render) + 'source?url=' + quote(search_url) - html = await get_url(search_url, 200, request_private_ip=True) + html = await get_url(webrender('source', quote(search_url)), 200, request_private_ip=True) Logger.debug(html) bs = BeautifulSoup(html, 'html.parser') results = bs.find_all('div', class_='result-item') diff --git a/modules/meme/jiki.py b/modules/meme/jiki.py index 8b5e4101..ff97fb20 100644 --- a/modules/meme/jiki.py +++ b/modules/meme/jiki.py @@ -1,13 +1,10 @@ from bs4 import BeautifulSoup -from config import CFG from core.builtins import Url from core.logger import Logger from core.utils.http import get_url from core.utils.i18n import Locale - -web_render = CFG.get_url('web_render') -web_render_local = CFG.get_url('web_render_local') +from core.utils.web_render import webrender async def jiki(term: str, locale: Locale): @@ -17,12 +14,7 @@ async def jiki(term: str, locale: Locale): :returns: 查询结果。''' try: api = 'https://jikipedia.com/search?phrase=' + term - if web_render: - use_local = True if web_render_local else False - else: - return - api = (web_render_local if use_local else web_render) + 'source?url=' + api - html = await get_url(api, 200) + html = await get_url(webrender('source', api), 200, request_private_ip=True) Logger.debug(html) bs = BeautifulSoup(html, 'html.parser') result = bs.select_one('[data-index="0"]') diff --git a/modules/meme/urban.py b/modules/meme/urban.py index f36567f4..375c24e4 100644 --- a/modules/meme/urban.py +++ b/modules/meme/urban.py @@ -1,13 +1,10 @@ import ujson as json -from config import CFG from core.builtins import Url from core.logger import Logger from core.utils.http import get_url from core.utils.i18n import Locale - -web_render = CFG.get_url('web_render') -web_render_local = CFG.get_url('web_render_local') +from core.utils.web_render import webrender async def urban(term: str, locale: Locale): @@ -17,12 +14,7 @@ async def urban(term: str, locale: Locale): :returns: 查询结果。''' try: url = 'http://api.urbandictionary.com/v0/define?term=' + term - if web_render: - use_local = True if web_render_local else False - else: - return - url = (web_render_local if use_local else web_render) + 'source?url=' + url - text = await get_url(url, 200, headers={'accept': '*/*', + text = await get_url(webrender('source', url), 200, headers={'accept': '*/*', 'accept-encoding': 'gzip, deflate', 'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,en-GB;q=0.6', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.62'}, diff --git a/modules/tweet/__init__.py b/modules/tweet/__init__.py index 15236859..f18a303c 100644 --- a/modules/tweet/__init__.py +++ b/modules/tweet/__init__.py @@ -1,16 +1,13 @@ import re import ujson as json -from config import CFG from core.builtins import Bot from core.builtins.message import Image, Url from core.component import module from core.dirty_check import check_bool, rickroll from core.logger import Logger from core.utils.http import download_to_cache, get_url - -web_render = CFG.get_url('web_render') -web_render_local = CFG.get_url('web_render_local') +from core.utils.web_render import webrender t = module('tweet', @@ -21,7 +18,7 @@ t = module('tweet', @t.handle(' {{tweet.help}}') -async def _(msg: Bot.MessageSession, tweet: str, use_local=True): +async def _(msg: Bot.MessageSession, tweet: str): if tweet.isdigit(): tweet_id = tweet else: @@ -31,11 +28,9 @@ async def _(msg: Bot.MessageSession, tweet: str, use_local=True): else: await msg.finish(msg.locale.t('tweet.message.invalid')) - if not web_render_local: - if not web_render: - Logger.warn('[Webrender] Webrender is not configured.') - await msg.finish(msg.locale.t("error.config.webrender.invalid")) - use_local = False + web_render = webrender('element_screenshot') + if not web_render: + await msg.finish(msg.locale.t("error.config.webrender.invalid")) res = await get_url(f'https://react-tweet.vercel.app/api/tweet/{tweet_id}', 200) res_json = json.loads(res) @@ -81,7 +76,7 @@ async def _(msg: Bot.MessageSession, tweet: str, use_local=True): } ''' - pic = await download_to_cache((web_render_local if use_local else web_render) + 'element_screenshot', method='POST', headers={ + pic = await download_to_cache(web_render, method='POST', headers={ 'Content-Type': 'application/json', }, post_data=json.dumps( {'url': f'https://react-tweet-next.vercel.app/light/{tweet_id}', 'css': css, 'mw': False, diff --git a/modules/wiki/inline.py b/modules/wiki/inline.py index a9f49ff1..9dc27f34 100644 --- a/modules/wiki/inline.py +++ b/modules/wiki/inline.py @@ -11,10 +11,11 @@ from core.logger import Logger from core.utils.http import download_to_cache from core.utils.image import svg_render from core.utils.image_table import image_table_render, ImageTable +from core.utils.web_render import WebRender from modules.wiki.utils.dbutils import WikiTargetInfo from modules.wiki.utils.screenshot_image import generate_screenshot_v1, generate_screenshot_v2 from modules.wiki.utils.wikilib import WikiLib -from .wiki import query_pages, generate_screenshot_v2_blocklist, web_render +from .wiki import query_pages, generate_screenshot_v2_blocklist wiki_inline = module('wiki_inline', desc='{wiki.help.wiki_inline.desc}', recommend_modules=['wiki'], @@ -147,7 +148,7 @@ async def _(msg: Bot.MessageSession): get_infobox = await generate_screenshot_v1(q[qq].realurl, qq, headers) if get_infobox: await msg.send_message(Image(get_infobox), quote=False) - if get_page.invalid_section and wiki_.wiki_info.in_allowlist and web_render: + if get_page.invalid_section and wiki_.wiki_info.in_allowlist and WebRender.status: i_msg_lst = [] if get_page.sections: session_data = [[str(i + 1), get_page.sections[i]] for i in diff --git a/modules/wiki/utils/screenshot_image.py b/modules/wiki/utils/screenshot_image.py index 50594964..6c88dcd5 100644 --- a/modules/wiki/utils/screenshot_image.py +++ b/modules/wiki/utils/screenshot_image.py @@ -9,12 +9,10 @@ import aiohttp import ujson as json from bs4 import BeautifulSoup, Comment -from config import CFG from core.logger import Logger from core.utils.http import download_to_cache +from core.utils.web_render import WebRender, webrender -web_render = CFG.get_url('web_render') -web_render_local = CFG.get_url('web_render_local') elements = ['.notaninfobox', '.portable-infobox', '.infobox', '.tpl-infobox', '.infoboxtable', '.infotemplatebox', '.skin-infobox', '.arcaeabox', '.moe-infobox', '.rotable'] assets_path = os.path.abspath('./assets/') @@ -25,10 +23,7 @@ async def generate_screenshot_v2(page_link, section=None, allow_special_page=Fal elements_ = elements.copy() if element and isinstance(element, List): elements_ += element - if not web_render_local: - if not web_render: - Logger.warn('[Webrender] Webrender is not configured.') - return False + if not WebRender.local use_local = False if not section: if allow_special_page and content_mode: @@ -37,7 +32,7 @@ async def generate_screenshot_v2(page_link, section=None, allow_special_page=Fal elements_.insert(0, '.diff') Logger.info('[Webrender] Generating element screenshot...') try: - img = await download_to_cache((web_render_local if use_local else web_render) + 'element_screenshot', + img = await download_to_cache(webrender('element_screenshot', use_local=use_local), status_code=200, headers={'Content-Type': 'application/json'}, method="POST", @@ -60,7 +55,7 @@ async def generate_screenshot_v2(page_link, section=None, allow_special_page=Fal else: Logger.info('[Webrender] Generating section screenshot...') try: - img = await download_to_cache((web_render_local if use_local else web_render) + 'section_screenshot', + img = await download_to_cache(webrender('element_screenshot', use_local=use_local), status_code=200, headers={'Content-Type': 'application/json'}, method="POST", @@ -84,11 +79,9 @@ async def generate_screenshot_v2(page_link, section=None, allow_special_page=Fal return img -async def generate_screenshot_v1(link, page_link, headers, section=None, allow_special_page=False) -> Union[str, bool]: - if not web_render_local: - if not web_render: - Logger.warn('[Webrender] Webrender is not configured.') - return False +async def generate_screenshot_v1(link, page_link, headers, use_local=True, section=None, allow_special_page=False) -> Union[str, bool]: + if not WebRender.local + use_local = False try: Logger.info('Starting find infobox/section..') if link[-1] != '/': @@ -340,7 +333,7 @@ async def generate_screenshot_v1(link, page_link, headers, section=None, allow_s os.remove(picname) try: async with aiohttp.ClientSession() as session: - async with session.post(web_render_local, headers={ + async with session.post(webrender(), headers={ 'Content-Type': 'application/json', }, data=json.dumps(html)) as resp: if resp.status != 200: @@ -349,15 +342,16 @@ async def generate_screenshot_v1(link, page_link, headers, section=None, allow_s with open(picname, 'wb+') as jpg: jpg.write(await resp.read()) except aiohttp.ClientConnectorError: - async with aiohttp.ClientSession() as session: - async with session.post(web_render, headers={ - 'Content-Type': 'application/json', - }, data=json.dumps(html)) as resp: - if resp.status != 200: - Logger.info(f'Failed to render: {await resp.text()}') - return False - with open(picname, 'wb+') as jpg: - jpg.write(await resp.read()) + if use_local: + async with aiohttp.ClientSession() as session: + async with session.post(webrender(use_local=False), headers={ + 'Content-Type': 'application/json', + }, data=json.dumps(html)) as resp: + if resp.status != 200: + Logger.info(f'Failed to render: {await resp.text()}') + return False + with open(picname, 'wb+') as jpg: + jpg.write(await resp.read()) return picname except Exception: traceback.print_exc() diff --git a/modules/wiki/utils/wikilib.py b/modules/wiki/utils/wikilib.py index 3a7cc034..0ebe84bc 100644 --- a/modules/wiki/utils/wikilib.py +++ b/modules/wiki/utils/wikilib.py @@ -8,7 +8,7 @@ from typing import Union, Dict, List import ujson as json import core.utils.html2text as html2text -from config import Config, CFG +from config import Config from core.builtins import Url from core.dirty_check import check from core.logger import Logger @@ -17,9 +17,8 @@ from core.utils.i18n import Locale, default_locale from core.exceptions import NoReportException from modules.wiki.utils.dbutils import WikiSiteInfo as DBSiteInfo, Audit from modules.wiki.utils.bot import BotAccount +from core.utils.web_render import webrender -web_render = CFG.get_url('web_render') -web_render_local = CFG.get_url('web_render_local') redirect_list = {'https://zh.moegirl.org.cn/api.php': 'https://mzh.moegirl.org.cn/api.php', # 萌娘百科强制使用移动版 API 'https://minecraft.fandom.com/api.php': 'https://minecraft.wiki/api.php', # no more Fandom then @@ -170,9 +169,7 @@ class WikiLib: request_local = False for x in request_by_web_render_list: if x.match(api): - if web_render: - use_local = True if web_render_local else False - api = (web_render_local if use_local else web_render) + 'source?url=' + urllib.parse.quote(api) + api = webrender('source', urllib.parse.quote(api)) request_local = True break diff --git a/modules/wiki/wiki.py b/modules/wiki/wiki.py index 6b1ccabe..f0d53062 100644 --- a/modules/wiki/wiki.py +++ b/modules/wiki/wiki.py @@ -4,7 +4,6 @@ from typing import Union import filetype -from config import CFG from core.builtins import Bot, Plain, Image, Voice, Url, confirm_command from core.types import MessageSession from core.utils.image_table import image_table_render, ImageTable @@ -13,11 +12,11 @@ from core.exceptions import AbuseWarning from core.logger import Logger from core.utils.http import download_to_cache from core.utils.image import svg_render +from core.utils.web_render import WebRender from modules.wiki.utils.dbutils import WikiTargetInfo from modules.wiki.utils.screenshot_image import generate_screenshot_v1, generate_screenshot_v2 from modules.wiki.utils.wikilib import WikiLib, WhatAreUDoingError, PageInfo, InvalidWikiError, QueryInfo -web_render = CFG.get_url('web_render') generate_screenshot_v2_blocklist = ['https://mzh.moegirl.org.cn', 'https://zh.moegirl.org.cn'] special_namespace = ['special', '特殊'] random_title = ['random', '随机页面', '隨機頁面'] @@ -208,7 +207,7 @@ async def query_pages(session: Union[Bot.MessageSession, QueryInfo], title: Unio plain_slice.append(session.locale.t('wiki.message.redirect', title=display_before_title, redirected_title=display_title)) if (r.link and r.selected_section and r.info.in_allowlist and - not r.invalid_section and web_render): + not r.invalid_section and WebRender.status): render_section_list.append( {r.link: {'url': r.info.realurl, 'section': r.selected_section, 'in_allowlist': r.info.in_allowlist}}) @@ -234,7 +233,7 @@ async def query_pages(session: Union[Bot.MessageSession, QueryInfo], title: Unio 'Template:Version disambiguation' in r.templates))}}) if plain_slice: msg_list.append(Plain('\n'.join(plain_slice))) - if web_render: + if WebRender.status: if r.invalid_section and r.info.in_allowlist: if isinstance(session, Bot.MessageSession) and session.Feature.image and r.sections: i_msg_lst = [] diff --git a/schedulers/mcv_rss.py b/schedulers/mcv_rss.py index 580af7bb..cafd0d99 100644 --- a/schedulers/mcv_rss.py +++ b/schedulers/mcv_rss.py @@ -7,7 +7,7 @@ import ujson as json from bs4 import BeautifulSoup from google_play_scraper import app as google_play_scraper -from config import CFG, Config +from config import Config from core.builtins import I18NContext, FormattedTime from core.logger import Logger from core.queue import JobQueue @@ -15,12 +15,10 @@ from core.scheduler import Scheduler, IntervalTrigger from core.utils.http import get_url from core.utils.ip import IP from core.utils.storedata import get_stored_list, update_stored_list - -web_render = CFG.get_url('web_render') -web_render_local = CFG.get_url('web_render_local') +from core.utils.web_render import webrender -async def get_article(version, use_local=True): +async def get_article(version): match_snapshot = re.match(r'.*?w.*', version) link = False if match_snapshot: @@ -43,15 +41,9 @@ async def get_article(version, use_local=True): + f'-release-candidate-{match_release_candidate.group(2)}' if not link: link = 'https://www.minecraft.net/en-us/article/minecraft-java-edition-' + re.sub("\\.", "-", version) - if not web_render_local: - if not web_render: - Logger.warn('[Webrender] Webrender is not configured.') - return '', '' - use_local = False - get = (web_render_local if use_local else web_render) + 'source?url=' + quote(link) try: - html = await get_url(get, attempt=1, request_private_ip=True, logging_err_resp=False) + html = await get_url(webrender('source', quote(link)), attempt=1, request_private_ip=True, logging_err_resp=False) soup = BeautifulSoup(html, 'html.parser') diff --git a/schedulers/minecraft_news.py b/schedulers/minecraft_news.py index b1ed7710..49593b2a 100644 --- a/schedulers/minecraft_news.py +++ b/schedulers/minecraft_news.py @@ -5,16 +5,14 @@ from urllib.parse import quote import ujson as json -from config import Config, CFG +from config import Config from core.builtins import Url, I18NContext from core.logger import Logger from core.queue import JobQueue from core.scheduler import Scheduler, IntervalTrigger from core.utils.http import get_url from core.utils.storedata import get_stored_list, update_stored_list - -web_render = CFG.get_url('web_render') -web_render_local = CFG.get_url('web_render_local') +from core.utils.web_render import webrender class Article: @@ -44,18 +42,12 @@ class Article: @Scheduler.scheduled_job(IntervalTrigger(seconds=60 if not Config('slower_schedule') else 180)) -async def start_check_news(use_local=True): +async def start_check_news(): baseurl = 'https://www.minecraft.net' url = quote( f'https://www.minecraft.net/content/minecraft-net/_jcr_content.articles.grid?tileselection=auto&tagsPath={",".join(Article.random_tags())}&offset=0&pageSize={Article.count}') - if not web_render_local: - if not web_render: - Logger.warn('[Webrender] Webrender is not configured.') - return - use_local = False try: - get = (web_render_local if use_local else web_render) + 'source?url=' + url - getpage = await get_url(get, 200, attempt=1, request_private_ip=True, logging_err_resp=False) + getpage = await get_url(webrender('source', url), 200, attempt=1, request_private_ip=True, logging_err_resp=False) if getpage: alist = get_stored_list('scheduler', 'mcnews') o_json = json.loads(getpage)