From 06533bd5d7ed91108b0c2dc125bb381fd1fc784e Mon Sep 17 00:00:00 2001 From: yzhh <2596322644@qq.com> Date: Sun, 12 Jun 2022 22:30:02 +0800 Subject: [PATCH] storing data into database --- bots/aiocqhttp/bot.py | 11 ++-- bots/aiocqhttp/message.py | 3 +- bots/aiogram/bot.py | 4 +- bots/aiogram/message.py | 3 +- bots/discord/bot.py | 5 +- bots/discord/message.py | 2 +- console.py | 5 +- core/console/template.py | 2 +- core/elements/message/__init__.py | 6 ++- core/utils/__init__.py | 1 + core/utils/bot.py | 6 +-- core/utils/storedata.py | 21 ++++++++ database/__init__.py | 31 ++++++++++- database/tables.py | 12 ++++- modules/mcv_rss/__init__.py | 87 +++++++++++++----------------- modules/minecraft_news/__init__.py | 31 +++-------- 16 files changed, 131 insertions(+), 99 deletions(-) create mode 100644 core/utils/storedata.py diff --git a/bots/aiocqhttp/bot.py b/bots/aiocqhttp/bot.py index 20538acf..f3cb9a2e 100644 --- a/bots/aiocqhttp/bot.py +++ b/bots/aiocqhttp/bot.py @@ -23,7 +23,7 @@ init() @bot.on_startup async def startup(): - await init_scheduler() + await init_scheduler(FetchTarget) bot.logger.setLevel(logging.WARNING) @@ -50,9 +50,10 @@ async def _(event: Event): msg = MessageSession(MsgInfo(targetId=targetId, senderId=f'QQ|{str(event.user_id)}', targetFrom='QQ|Group' if event.detail_type == 'group' else 'QQ', - senderFrom='QQ', senderName=''), Session(message=event, - target=event.group_id if event.detail_type == 'group' else event.user_id, - sender=event.user_id)) + senderFrom='QQ', senderName='', clientName='QQ'), + Session(message=event, + target=event.group_id if event.detail_type == 'group' else event.user_id, + sender=event.user_id)) await parser(msg, running_mention=True) @@ -76,7 +77,7 @@ async def _(event): msg = MessageSessionGuild(MsgInfo(targetId=f'QQ|Guild|{str(event.guild_id)}|{str(event.channel_id)}', senderId=f'QQ|Tiny|{str(event.user_id)}', targetFrom='QQ|Guild', - senderFrom='QQ|Tiny', senderName=''), + senderFrom='QQ|Tiny', senderName='', clientName='QQ'), Session(message=event, target=f'{str(event.guild_id)}|{str(event.channel_id)}', sender=event.user_id)) diff --git a/bots/aiocqhttp/message.py b/bots/aiocqhttp/message.py index a833a9bf..c8fdaecc 100644 --- a/bots/aiocqhttp/message.py +++ b/bots/aiocqhttp/message.py @@ -164,7 +164,8 @@ class FetchedSession(FS): senderId=f'{targetFrom}|{targetId}', targetFrom=targetFrom, senderFrom=targetFrom, - senderName='') + senderName='', + clientName='QQ') self.session = Session(message=False, target=targetId, sender=targetId) if targetFrom == 'QQ|Guild': self.parent = MessageSessionGuild(self.target, self.session) diff --git a/bots/aiogram/bot.py b/bots/aiogram/bot.py index 81b4e3ec..d1b4803e 100644 --- a/bots/aiogram/bot.py +++ b/bots/aiogram/bot.py @@ -25,13 +25,13 @@ async def msg_handler(message: types.Message): msg = MessageSession(MsgInfo(targetId=f'Telegram|{message.chat.type}|{message.chat.id}', senderId=f'Telegram|User|{message.from_user.id}', targetFrom=f'Telegram|{message.chat.type}', - senderFrom='Telegram|User', senderName=message.from_user.username), + senderFrom='Telegram|User', senderName=message.from_user.username, clientName='Telegram'), Session(message=message, target=message.chat.id, sender=message.from_user.id)) await parser(msg) async def on_startup(dispatcher): - await init_scheduler() + await init_scheduler(FetchTarget) await load_prompt(FetchTarget) diff --git a/bots/aiogram/message.py b/bots/aiogram/message.py index 491ca0ba..c71f09f1 100644 --- a/bots/aiogram/message.py +++ b/bots/aiogram/message.py @@ -138,7 +138,8 @@ class FetchedSession(FS): senderId=f'{targetFrom}|{targetId}', targetFrom=targetFrom, senderFrom=targetFrom, - senderName='') + senderName='', + clientName='Telegram') self.session = Session(message=False, target=targetId, sender=targetId) self.parent = MessageSession(self.target, self.session) diff --git a/bots/discord/bot.py b/bots/discord/bot.py index 98b49599..72927f67 100644 --- a/bots/discord/bot.py +++ b/bots/discord/bot.py @@ -22,7 +22,7 @@ async def on_ready(): Logger.info('Logged on as ' + str(client.user)) global count if count == 0: - await init_scheduler() + await init_scheduler(FetchTarget) await load_prompt(FetchTarget) count = 1 @@ -37,7 +37,8 @@ async def on_message(message): target = "Discord|DM|Channel" msg = MessageSession(target=MsgInfo(targetId=f"{target}|{message.channel.id}", senderId=f"Discord|Client|{message.author.id}", - senderName=message.author.name, targetFrom=target, senderFrom="Discord|Client"), + senderName=message.author.name, targetFrom=target, senderFrom="Discord|Client", + clientName='Discord'), session=Session(message=message, target=message.channel, sender=message.author)) await parser(msg) diff --git a/bots/discord/message.py b/bots/discord/message.py index 94fdfc24..6969de81 100644 --- a/bots/discord/message.py +++ b/bots/discord/message.py @@ -164,7 +164,7 @@ class FetchedSession(FS): senderId=f'{targetFrom}|{targetId}', targetFrom=targetFrom, senderFrom=targetFrom, - senderName='') + senderName='', clientName='Discord') self.session = Session(message=False, target=targetId, sender=targetId) self.parent = MessageSession(self.target, self.session) diff --git a/console.py b/console.py index 3db32189..521ce1c1 100644 --- a/console.py +++ b/console.py @@ -16,9 +16,8 @@ from datetime import datetime from bot import init_bot from core.elements import MsgInfo, Session, PrivateAssets, EnableDirtyWordCheck -from core.console.template import Template as MessageSession +from core.console.template import Template as MessageSession, FetchTarget from core.parser.message import parser -from core.scheduler import Scheduler from core.utils import init, init_scheduler from core.logger import Logger @@ -28,7 +27,7 @@ init() async def console_scheduler(): - await init_scheduler() + await init_scheduler(FetchTarget) async def console_command(): diff --git a/core/console/template.py b/core/console/template.py index 62d05279..a0ebbbfb 100644 --- a/core/console/template.py +++ b/core/console/template.py @@ -96,7 +96,7 @@ class FetchedSession(FS): senderId=f'{targetFrom}|{targetId}', targetFrom=targetFrom, senderFrom=targetFrom, - senderName='') + senderName='', clientName='TEST') self.session = Session(message=False, target=targetId, sender=targetId) self.parent = Template(self.target, self.session) diff --git a/core/elements/message/__init__.py b/core/elements/message/__init__.py index e078d4c6..0c6f1212 100644 --- a/core/elements/message/__init__.py +++ b/core/elements/message/__init__.py @@ -5,20 +5,22 @@ from core.exceptions import FinishedException class MsgInfo: - __slots__ = ["targetId", "senderId", "senderName", "targetFrom", "senderInfo", "senderFrom"] + __slots__ = ["targetId", "senderId", "senderName", "targetFrom", "senderInfo", "senderFrom", "clientName"] def __init__(self, targetId: [int, str], senderId: [int, str], senderName: str, targetFrom: str, - senderFrom: str + senderFrom: str, + clientName: str ): self.targetId = targetId self.senderId = senderId self.senderName = senderName self.targetFrom = targetFrom self.senderFrom = senderFrom + self.clientName = clientName class Session: diff --git a/core/utils/__init__.py b/core/utils/__init__.py index 7f2a556c..beaef390 100644 --- a/core/utils/__init__.py +++ b/core/utils/__init__.py @@ -4,3 +4,4 @@ from .http import * from .image_table import * from .message import * from .message import * +from .storedata import * diff --git a/core/utils/bot.py b/core/utils/bot.py index f7eb3c6b..78758634 100644 --- a/core/utils/bot.py +++ b/core/utils/bot.py @@ -28,14 +28,14 @@ def init() -> None: write_tag.close() -async def init_scheduler() -> None: +async def init_scheduler(ft) -> None: gather_list = [] Modules = ModulesManager.return_modules_list_as_dict() for x in Modules: if isinstance(Modules[x], StartUp): - gather_list.append(asyncio.ensure_future(Modules[x].function(FetchTarget))) + gather_list.append(asyncio.ensure_future(Modules[x].function(ft))) elif isinstance(Modules[x], Schedule): - Scheduler.add_job(func=Modules[x].function, trigger=Modules[x].trigger, args=[FetchTarget], misfire_grace_time=30, max_instance=1) + Scheduler.add_job(func=Modules[x].function, trigger=Modules[x].trigger, args=[ft], misfire_grace_time=30, max_instance=1) await asyncio.gather(*gather_list) Scheduler.start() logging.getLogger('apscheduler.executors.default').setLevel(logging.WARNING) diff --git a/core/utils/storedata.py b/core/utils/storedata.py new file mode 100644 index 00000000..73e207ee --- /dev/null +++ b/core/utils/storedata.py @@ -0,0 +1,21 @@ +import ujson as json + +from core.elements import FetchTarget + +from database import BotDBUtil + + +def get_stored_list(bot: FetchTarget, name): + get = BotDBUtil.Data(bot).get(name=name) + if get is None: + return [] + else: + return json.loads(get.value) + + +def update_stored_list(bot: FetchTarget, name, value): + edit = BotDBUtil.Data(bot).update(name=name, value=json.dumps(value)) + return edit + + +__all__ = ['get_stored_list', 'update_stored_list'] diff --git a/database/__init__.py b/database/__init__.py index cb03f636..a99a4853 100644 --- a/database/__init__.py +++ b/database/__init__.py @@ -1,12 +1,13 @@ import datetime +from typing import Union from tenacity import retry, stop_after_attempt from config import Config -from core.elements.message import MessageSession +from core.elements.message import MessageSession, FetchTarget from core.elements.temp import EnabledModulesCache, SenderInfoCache from database.orm import DBSession -from database.tables import EnabledModules, MuteList, SenderInfo, TargetAdmin, CommandTriggerTime, GroupAllowList +from database.tables import EnabledModules, MuteList, SenderInfo, TargetAdmin, CommandTriggerTime, GroupAllowList, StoredData cache = Config('db_cache') @@ -254,5 +255,31 @@ class BotDBUtil: session.delete(self.query) session.commit() + class Data: + def __init__(self, msg: Union[MessageSession, FetchTarget]): + self.targetName = msg.target.clientName if isinstance(msg, MessageSession) else msg.name + + @retry(stop=stop_after_attempt(3)) + @auto_rollback_error + def add(self, name, value: str): + session.add(StoredData(name=f'{self.targetName}|{name}', value=value)) + session.commit() + + @retry(stop=stop_after_attempt(3)) + @auto_rollback_error + def get(self, name): + return session.query(StoredData).filter_by(name=f'{self.targetName}|{name}').first() + + @retry(stop=stop_after_attempt(3)) + @auto_rollback_error + def update(self, name, value: str): + exists = self.get(name) + if exists is None: + self.add(name=name, value=value) + else: + exists.value = value + session.commit() + return True + __all__ = ["BotDBUtil", "auto_rollback_error", "session"] diff --git a/database/tables.py b/database/tables.py index a0c4d062..7464b002 100644 --- a/database/tables.py +++ b/database/tables.py @@ -1,6 +1,9 @@ from sqlalchemy import Column, Integer, String, Text, TIMESTAMP, Boolean, text +from sqlalchemy.dialects.mysql import LONGTEXT from sqlalchemy.ext.declarative import declarative_base +from config import Config + Base = declarative_base() @@ -22,6 +25,13 @@ class SenderInfo(Base): disable_typing = Column(Boolean, default=False) +class StoredData(Base): + """数据存储""" + __tablename__ = "StoredData" + name = Column(String(512), primary_key=True) + value = Column(LONGTEXT if Config('db_path').startswith('mysql') else Text) + + class TargetAdmin(Base): """所属赋予的管理员""" __tablename__ = "TargetAdmin" @@ -49,4 +59,4 @@ class GroupAllowList(Base): targetId = Column(String(512), primary_key=True) -__all__ = ["Base", "EnabledModules", "TargetAdmin", "SenderInfo", "CommandTriggerTime", "GroupAllowList"] +__all__ = ["Base", "EnabledModules", "TargetAdmin", "SenderInfo", "CommandTriggerTime", "GroupAllowList", "StoredData"] diff --git a/modules/mcv_rss/__init__.py b/modules/mcv_rss/__init__.py index 3283fd9d..c2bfe418 100644 --- a/modules/mcv_rss/__init__.py +++ b/modules/mcv_rss/__init__.py @@ -11,17 +11,7 @@ from config import Config from core.component import on_schedule from core.elements import FetchTarget, IntervalTrigger, PrivateAssets from core.logger import Logger -from core.utils import get_url - - -def getfileversions(path): - if not os.path.exists(path): - a = open(path, 'a', encoding='utf-8') - a.close() - w = open(path, 'r+', encoding='utf-8') - s = w.read().split('\n') - w.close() - return s +from core.utils import get_stored_list, update_stored_list, get_url async def get_article(version): @@ -75,38 +65,32 @@ trigger_times = 60 if not Config('slower_schedule') else 180 async def mcv_rss(bot: FetchTarget): url = 'http://launchermeta.mojang.com/mc/game/version_manifest.json' try: - version_file = os.path.abspath(f'{PrivateAssets.path}/mcversion.txt') - verlist = getfileversions(version_file) + verlist = get_stored_list(bot, 'mcv_rss') file = json.loads(await get_url(url)) release = file['latest']['release'] snapshot = file['latest']['snapshot'] if release not in verlist: Logger.info(f'huh, we find {release}.') await bot.post_message('mcv_rss', '启动器已更新' + file['latest']['release'] + '正式版。') - addversion = open(version_file, 'a', encoding='utf-8') - addversion.write('\n' + release) - addversion.close() - verlist = getfileversions(version_file) + verlist.append(release) + update_stored_list(bot, 'mcv_rss', verlist) article = await get_article(release) if article[0] != '': await bot.post_message('minecraft_news', f'Minecraft官网发布了{release}的更新日志：\n' + article[0]) - newsfile = os.path.abspath(f'{PrivateAssets.path}/mcnews.txt') - addnews = open(newsfile, 'a', encoding='utf-8') - addnews.write('\n' + article[1]) - addnews.close() + get_stored_news_title = get_stored_list(bot, 'mcnews') + get_stored_news_title.append(article[1]) + update_stored_list(bot, 'mcname', get_stored_news_title) if snapshot not in verlist: Logger.info(f'huh, we find {snapshot}.') await bot.post_message('mcv_rss', '启动器已更新' + file['latest']['snapshot'] + '快照。') - addversion = open(version_file, 'a', encoding='utf-8') - addversion.write('\n' + snapshot) - addversion.close() + verlist.append(snapshot) + update_stored_list(bot, 'mcv_rss', verlist) article = await get_article(snapshot) if article[0] != '': await bot.post_message('minecraft_news', f'Minecraft官网发布了{snapshot}的更新日志：\n' + article[0]) - newsfile = os.path.abspath(f'{PrivateAssets.path}/mcnews.txt') - addnews = open(newsfile, 'a', encoding='utf-8') - addnews.write('\n' + article[1]) - addnews.close() + get_stored_news_title = get_stored_list(bot, 'mcnews') + get_stored_news_title.append(article[1]) + update_stored_list(bot, 'mcname', get_stored_news_title) except Exception: traceback.print_exc() @@ -117,15 +101,13 @@ async def mcv_rss(bot: FetchTarget): desc='开启后当Minecraft基岩版商店更新时将会自动推送消息。', alias='mcbvrss') async def mcbv_rss(bot: FetchTarget): try: - version_file = os.path.abspath(f'{PrivateAssets.path}/mcbeversion.txt') - verlist = getfileversions(version_file) + verlist = get_stored_list(bot, 'mcbv_rss') version = google_play_scraper('com.mojang.minecraftpe')['version'] if version not in verlist: Logger.info(f'huh, we find bedrock {version}.') await bot.post_message('mcbv_rss', '基岩版商店已更新' + version + '正式版。') - addversion = open(version_file, 'a', encoding='utf-8') - addversion.write('\n' + version) - addversion.close() + verlist.append(version) + update_stored_list(bot, 'mcbv_rss', verlist) except Exception: traceback.print_exc() @@ -136,23 +118,24 @@ async def mcbv_rss(bot: FetchTarget): desc='开启后当Jira更新Java版时将会自动推送消息。', alias='mcvjirarss') async def mcv_jira_rss(bot: FetchTarget): try: - version_file = os.path.abspath(f'{PrivateAssets.path}/mcjira_Java.txt') - verlist = getfileversions(version_file) + verlist = get_stored_list(bot, 'mcv_jira_rss') file = json.loads(await get_url('https://bugs.mojang.com/rest/api/2/project/10400/versions')) releases = [] for v in file: if not v['archived']: releases.append(v['name']) + else: + if v['name'] not in verlist: + verlist.append(v['name']) for release in releases: if release not in verlist: Logger.info(f'huh, we find {release}.') - verlist.append(release) await bot.post_message('mcv_jira_rss', f'Jira已更新Java版 {release}。' f'\n（Jira上的信息仅作版本号预览用，不代表启动器已更新此版本）') - addversion = open(version_file, 'a', encoding='utf-8') - addversion.write('\n' + release) - addversion.close() + verlist.append(release) + update_stored_list(bot, 'mcv_jira_rss', verlist) + except Exception: traceback.print_exc() @@ -164,23 +147,24 @@ async def mcv_jira_rss(bot: FetchTarget): desc='开启后当Jira更新基岩版时将会自动推送消息。', alias='mcbvjirarss') async def mcbv_jira_rss(bot: FetchTarget): try: - version_file = os.path.abspath(f'{PrivateAssets.path}/mcjira_Bedrock.txt') - verlist = getfileversions(version_file) + verlist = get_stored_list(bot, 'mcbv_jira_rss') file = json.loads(await get_url('https://bugs.mojang.com/rest/api/2/project/10200/versions')) releases = [] for v in file: if not v['archived']: releases.append(v['name']) + else: + if v['name'] not in verlist: + verlist.append(v['name']) for release in releases: if release not in verlist: Logger.info(f'huh, we find {release}.') - verlist.append(release) + await bot.post_message('mcbv_jira_rss', f'Jira已更新基岩版 {release}。' f'\n（Jira上的信息仅作版本号预览用，不代表商城已更新此版本）') - addversion = open(version_file, 'a', encoding='utf-8') - addversion.write('\n' + release) - addversion.close() + verlist.append(release) + update_stored_list(bot, 'mcbv_jira_rss', verlist) except Exception: traceback.print_exc() @@ -192,22 +176,23 @@ async def mcbv_jira_rss(bot: FetchTarget): desc='开启后当Jira更新Dungeons版本时将会自动推送消息。', alias='mcdvjirarss') async def mcdv_jira_rss(bot: FetchTarget): try: - version_file = os.path.abspath(f'{PrivateAssets.path}/mcjira_Minecraft Dungeons.txt') - verlist = getfileversions(version_file) + verlist = get_stored_list(bot, 'mcdv_jira_rss') file = json.loads(await get_url('https://bugs.mojang.com/rest/api/2/project/11901/versions')) releases = [] for v in file: if not v['archived']: releases.append(v['name']) + else: + if v['name'] not in verlist: + verlist.append(v['name']) for release in releases: if release not in verlist: Logger.info(f'huh, we find {release}.') - verlist.append(release) + await bot.post_message('mcdv_jira_rss', f'Jira已更新Minecraft Dungeons {release}。' f'\n（Jira上的信息仅作版本号预览用，不代表启动器/商城已更新此版本）') - addversion = open(version_file, 'a', encoding='utf-8') - addversion.write('\n' + release) - addversion.close() + verlist.append(release) + update_stored_list(bot, 'mcdv_jira_rss', verlist) except Exception: traceback.print_exc() diff --git a/modules/minecraft_news/__init__.py b/modules/minecraft_news/__init__.py index 441d54ff..00b9a833 100644 --- a/modules/minecraft_news/__init__.py +++ b/modules/minecraft_news/__init__.py @@ -10,17 +10,7 @@ from config import Config from core.component import on_schedule from core.elements import FetchTarget, IntervalTrigger, PrivateAssets, Url from core.logger import Logger -from core.utils import get_url - - -def getfileversions(path): - if not os.path.exists(path): - a = open(path, 'a', encoding='utf-8') - a.close() - w = open(path, 'r+', encoding='utf-8') - s = w.read().split('\n') - w.close() - return s +from core.utils import get_url, get_stored_list, update_stored_list class Article: @@ -53,7 +43,6 @@ class Article: recommend_modules=['feedback_news'], trigger=IntervalTrigger(seconds=60 if not Config('slower_schedule') else 180), desc='开启后将会自动推送来自Minecraft官网的新闻。') async def start_check_news(bot: FetchTarget): - file_ = os.path.abspath(f'{PrivateAssets.path}/mcnews.txt') baseurl = 'https://www.minecraft.net' url = quote( f'https://www.minecraft.net/content/minecraft-net/_jcr_content.articles.grid?tileselection=auto&tagsPath={",".join(Article.random_tags())}&offset=0&pageSize={Article.count}') @@ -63,7 +52,7 @@ async def start_check_news(bot: FetchTarget): return getpage = await get_url(get) if getpage: - alist = getfileversions(file_) + alist = get_stored_list(bot, 'mcnews') o_json = json.loads(getpage) o_nws = o_json['article_grid'] Article.count = o_json['article_count'] @@ -78,9 +67,8 @@ async def start_check_news(bot: FetchTarget): now = datetime.now() if now - publish_date < timedelta(days=2): await bot.post_message('minecraft_news', articletext) - addversion = open(file_, 'a', encoding='utf-8') - addversion.write('\n' + title) - addversion.close() + alist.append(title) + update_stored_list(bot, 'mcnews', alist) @on_schedule('feedback_news', developers=['Dianliang233'], recommend_modules=['minecraft_news'], @@ -92,9 +80,7 @@ async def feedback_news(bot: FetchTarget): 'url': 'https://minecraftfeedback.zendesk.com/api/v2/help_center/en-us/sections/360001186971/articles?per_page=5'}] for section in sections: try: - name = section['name'] - version_file = os.path.abspath(f'{PrivateAssets.path}/feedback_{name}.txt') - alist = getfileversions(version_file) + alist = get_stored_list(bot, 'mcfeedbacknews') get = await get_url(section['url']) res = json.loads(get) articles = [] @@ -105,12 +91,9 @@ async def feedback_news(bot: FetchTarget): name = article['name'] link = article['html_url'] Logger.info(f'huh, we find {name}.') - alist.append(name) await bot.post_message('feedback_news', f'Minecraft Feedback 发布了新的文章：\n{name}\n{str(Url(link))}') - addversion = open(version_file, 'a', encoding='utf-8') - addversion.write('\n' + name) - addversion.close() + alist.append(name) + update_stored_list(bot, 'mcfeedbacknews', alist) except Exception: traceback.print_exc() - print(get)