xtex/akari-bot
Archived
1
0
Fork 0
Code Issues Pull requests Activity

storing data into database

Browse source
This commit is contained in:
yzhh 2022-06-12 22:30:02 +08:00
parent 49ca216a8a
commit 06533bd5d7
16 changed files with 131 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
bots/aiocqhttp/bot.py
View file

@ -23,7 +23,7 @@ init()
@bot.on_startup
async def startup():
await init_scheduler()
await init_scheduler(FetchTarget)
bot.logger.setLevel(logging.WARNING)
@ -50,9 +50,10 @@ async def _(event: Event):
msg = MessageSession(MsgInfo(targetId=targetId,
senderId=f'QQ|{str(event.user_id)}',
targetFrom='QQ|Group' if event.detail_type == 'group' else 'QQ',
senderFrom='QQ', senderName=''), Session(message=event,
target=event.group_id if event.detail_type == 'group' else event.user_id,
sender=event.user_id))
senderFrom='QQ', senderName='', clientName='QQ'),
Session(message=event,
target=event.group_id if event.detail_type == 'group' else event.user_id,
sender=event.user_id))
await parser(msg, running_mention=True)
@ -76,7 +77,7 @@ async def _(event):
msg = MessageSessionGuild(MsgInfo(targetId=f'QQ|Guild|{str(event.guild_id)}|{str(event.channel_id)}',
senderId=f'QQ|Tiny|{str(event.user_id)}',
targetFrom='QQ|Guild',
senderFrom='QQ|Tiny', senderName=''),
senderFrom='QQ|Tiny', senderName='', clientName='QQ'),
Session(message=event,
target=f'{str(event.guild_id)}|{str(event.channel_id)}',
sender=event.user_id))

3
bots/aiocqhttp/message.py
View file

@ -164,7 +164,8 @@ class FetchedSession(FS):
senderId=f'{targetFrom}|{targetId}',
targetFrom=targetFrom,
senderFrom=targetFrom,
senderName='')
senderName='',
clientName='QQ')
self.session = Session(message=False, target=targetId, sender=targetId)
if targetFrom == 'QQ|Guild':
self.parent = MessageSessionGuild(self.target, self.session)

4
bots/aiogram/bot.py
View file

@ -25,13 +25,13 @@ async def msg_handler(message: types.Message):
msg = MessageSession(MsgInfo(targetId=f'Telegram|{message.chat.type}|{message.chat.id}',
senderId=f'Telegram|User|{message.from_user.id}',
targetFrom=f'Telegram|{message.chat.type}',
senderFrom='Telegram|User', senderName=message.from_user.username),
senderFrom='Telegram|User', senderName=message.from_user.username, clientName='Telegram'),
Session(message=message, target=message.chat.id, sender=message.from_user.id))
await parser(msg)
async def on_startup(dispatcher):
await init_scheduler()
await init_scheduler(FetchTarget)
await load_prompt(FetchTarget)

3
bots/aiogram/message.py
View file

@ -138,7 +138,8 @@ class FetchedSession(FS):
senderId=f'{targetFrom}|{targetId}',
targetFrom=targetFrom,
senderFrom=targetFrom,
senderName='')
senderName='',
clientName='Telegram')
self.session = Session(message=False, target=targetId, sender=targetId)
self.parent = MessageSession(self.target, self.session)

5
bots/discord/bot.py
View file

@ -22,7 +22,7 @@ async def on_ready():
Logger.info('Logged on as ' + str(client.user))
global count
if count == 0:
await init_scheduler()
await init_scheduler(FetchTarget)
await load_prompt(FetchTarget)
count = 1
@ -37,7 +37,8 @@ async def on_message(message):
target = "Discord|DM|Channel"
msg = MessageSession(target=MsgInfo(targetId=f"{target}|{message.channel.id}",
senderId=f"Discord|Client|{message.author.id}",
senderName=message.author.name, targetFrom=target, senderFrom="Discord|Client"),
senderName=message.author.name, targetFrom=target, senderFrom="Discord|Client",
clientName='Discord'),
session=Session(message=message, target=message.channel, sender=message.author))
await parser(msg)

2
bots/discord/message.py
View file

@ -164,7 +164,7 @@ class FetchedSession(FS):
senderId=f'{targetFrom}|{targetId}',
targetFrom=targetFrom,
senderFrom=targetFrom,
senderName='')
senderName='', clientName='Discord')
self.session = Session(message=False, target=targetId, sender=targetId)
self.parent = MessageSession(self.target, self.session)

5
console.py
View file

@ -16,9 +16,8 @@ from datetime import datetime
from bot import init_bot
from core.elements import MsgInfo, Session, PrivateAssets, EnableDirtyWordCheck
from core.console.template import Template as MessageSession
from core.console.template import Template as MessageSession, FetchTarget
from core.parser.message import parser
from core.scheduler import Scheduler
from core.utils import init, init_scheduler
from core.logger import Logger
@ -28,7 +27,7 @@ init()
async def console_scheduler():
await init_scheduler()
await init_scheduler(FetchTarget)
async def console_command():

2
core/console/template.py
View file

@ -96,7 +96,7 @@ class FetchedSession(FS):
senderId=f'{targetFrom}|{targetId}',
targetFrom=targetFrom,
senderFrom=targetFrom,
senderName='')
senderName='', clientName='TEST')
self.session = Session(message=False, target=targetId, sender=targetId)
self.parent = Template(self.target, self.session)

6
core/elements/message/__init__.py
View file

@ -5,20 +5,22 @@ from core.exceptions import FinishedException
class MsgInfo:
__slots__ = ["targetId", "senderId", "senderName", "targetFrom", "senderInfo", "senderFrom"]
__slots__ = ["targetId", "senderId", "senderName", "targetFrom", "senderInfo", "senderFrom", "clientName"]
def __init__(self,
targetId: [int, str],
senderId: [int, str],
senderName: str,
targetFrom: str,
senderFrom: str
senderFrom: str,
clientName: str
):
self.targetId = targetId
self.senderId = senderId
self.senderName = senderName
self.targetFrom = targetFrom
self.senderFrom = senderFrom
self.clientName = clientName
class Session:

1
core/utils/__init__.py
View file

@ -4,3 +4,4 @@ from .http import *
from .image_table import *
from .message import *
from .message import *
from .storedata import *

6
core/utils/bot.py
View file

@ -28,14 +28,14 @@ def init() -> None:
write_tag.close()
async def init_scheduler() -> None:
async def init_scheduler(ft) -> None:
gather_list = []
Modules = ModulesManager.return_modules_list_as_dict()
for x in Modules:
if isinstance(Modules[x], StartUp):
gather_list.append(asyncio.ensure_future(Modules[x].function(FetchTarget)))
gather_list.append(asyncio.ensure_future(Modules[x].function(ft)))
elif isinstance(Modules[x], Schedule):
Scheduler.add_job(func=Modules[x].function, trigger=Modules[x].trigger, args=[FetchTarget], misfire_grace_time=30, max_instance=1)
Scheduler.add_job(func=Modules[x].function, trigger=Modules[x].trigger, args=[ft], misfire_grace_time=30, max_instance=1)
await asyncio.gather(*gather_list)
Scheduler.start()
logging.getLogger('apscheduler.executors.default').setLevel(logging.WARNING)

21
core/utils/storedata.py Normal file
View file

@ -0,0 +1,21 @@
import ujson as json
from core.elements import FetchTarget
from database import BotDBUtil
def get_stored_list(bot: FetchTarget, name):
get = BotDBUtil.Data(bot).get(name=name)
if get is None:
return []
else:
return json.loads(get.value)
def update_stored_list(bot: FetchTarget, name, value):
edit = BotDBUtil.Data(bot).update(name=name, value=json.dumps(value))
return edit
__all__ = ['get_stored_list', 'update_stored_list']

31
database/__init__.py
View file

@ -1,12 +1,13 @@
import datetime
from typing import Union
from tenacity import retry, stop_after_attempt
from config import Config
from core.elements.message import MessageSession
from core.elements.message import MessageSession, FetchTarget
from core.elements.temp import EnabledModulesCache, SenderInfoCache
from database.orm import DBSession
from database.tables import EnabledModules, MuteList, SenderInfo, TargetAdmin, CommandTriggerTime, GroupAllowList
from database.tables import EnabledModules, MuteList, SenderInfo, TargetAdmin, CommandTriggerTime, GroupAllowList, StoredData
cache = Config('db_cache')
@ -254,5 +255,31 @@ class BotDBUtil:
session.delete(self.query)
session.commit()
class Data:
def __init__(self, msg: Union[MessageSession, FetchTarget]):
self.targetName = msg.target.clientName if isinstance(msg, MessageSession) else msg.name
@retry(stop=stop_after_attempt(3))
@auto_rollback_error
def add(self, name, value: str):
session.add(StoredData(name=f'{self.targetName}|{name}', value=value))
session.commit()
@retry(stop=stop_after_attempt(3))
@auto_rollback_error
def get(self, name):
return session.query(StoredData).filter_by(name=f'{self.targetName}|{name}').first()
@retry(stop=stop_after_attempt(3))
@auto_rollback_error
def update(self, name, value: str):
exists = self.get(name)
if exists is None:
self.add(name=name, value=value)
else:
exists.value = value
session.commit()
return True
__all__ = ["BotDBUtil", "auto_rollback_error", "session"]

12
database/tables.py
View file

@ -1,6 +1,9 @@
from sqlalchemy import Column, Integer, String, Text, TIMESTAMP, Boolean, text
from sqlalchemy.dialects.mysql import LONGTEXT
from sqlalchemy.ext.declarative import declarative_base
from config import Config
Base = declarative_base()
@ -22,6 +25,13 @@ class SenderInfo(Base):
disable_typing = Column(Boolean, default=False)
class StoredData(Base):
"""数据存储"""
__tablename__ = "StoredData"
name = Column(String(512), primary_key=True)
value = Column(LONGTEXT if Config('db_path').startswith('mysql') else Text)
class TargetAdmin(Base):
"""所属赋予的管理员"""
__tablename__ = "TargetAdmin"
@ -49,4 +59,4 @@ class GroupAllowList(Base):
targetId = Column(String(512), primary_key=True)
__all__ = ["Base", "EnabledModules", "TargetAdmin", "SenderInfo", "CommandTriggerTime", "GroupAllowList"]
__all__ = ["Base", "EnabledModules", "TargetAdmin", "SenderInfo", "CommandTriggerTime", "GroupAllowList", "StoredData"]

87
modules/mcv_rss/__init__.py
View file

@ -11,17 +11,7 @@ from config import Config
from core.component import on_schedule
from core.elements import FetchTarget, IntervalTrigger, PrivateAssets
from core.logger import Logger
from core.utils import get_url
def getfileversions(path):
if not os.path.exists(path):
a = open(path, 'a', encoding='utf-8')
a.close()
w = open(path, 'r+', encoding='utf-8')
s = w.read().split('\n')
w.close()
return s
from core.utils import get_stored_list, update_stored_list, get_url
async def get_article(version):
@ -75,38 +65,32 @@ trigger_times = 60 if not Config('slower_schedule') else 180
async def mcv_rss(bot: FetchTarget):
url = 'http://launchermeta.mojang.com/mc/game/version_manifest.json'
try:
version_file = os.path.abspath(f'{PrivateAssets.path}/mcversion.txt')
verlist = getfileversions(version_file)
verlist = get_stored_list(bot, 'mcv_rss')
file = json.loads(await get_url(url))
release = file['latest']['release']
snapshot = file['latest']['snapshot']
if release not in verlist:
Logger.info(f'huh, we find {release}.')
await bot.post_message('mcv_rss', '启动器已更新' + file['latest']['release'] + '正式版。')
addversion = open(version_file, 'a', encoding='utf-8')
addversion.write('\n' + release)
addversion.close()
verlist = getfileversions(version_file)
verlist.append(release)
update_stored_list(bot, 'mcv_rss', verlist)
article = await get_article(release)
if article[0] != '':
await bot.post_message('minecraft_news', f'Minecraft官网发布了{release}的更新日志：\n' + article[0])
newsfile = os.path.abspath(f'{PrivateAssets.path}/mcnews.txt')
addnews = open(newsfile, 'a', encoding='utf-8')
addnews.write('\n' + article[1])
addnews.close()
get_stored_news_title = get_stored_list(bot, 'mcnews')
get_stored_news_title.append(article[1])
update_stored_list(bot, 'mcname', get_stored_news_title)
if snapshot not in verlist:
Logger.info(f'huh, we find {snapshot}.')
await bot.post_message('mcv_rss', '启动器已更新' + file['latest']['snapshot'] + '快照。')
addversion = open(version_file, 'a', encoding='utf-8')
addversion.write('\n' + snapshot)
addversion.close()
verlist.append(snapshot)
update_stored_list(bot, 'mcv_rss', verlist)
article = await get_article(snapshot)
if article[0] != '':
await bot.post_message('minecraft_news', f'Minecraft官网发布了{snapshot}的更新日志：\n' + article[0])
newsfile = os.path.abspath(f'{PrivateAssets.path}/mcnews.txt')
addnews = open(newsfile, 'a', encoding='utf-8')
addnews.write('\n' + article[1])
addnews.close()
get_stored_news_title = get_stored_list(bot, 'mcnews')
get_stored_news_title.append(article[1])
update_stored_list(bot, 'mcname', get_stored_news_title)
except Exception:
traceback.print_exc()
@ -117,15 +101,13 @@ async def mcv_rss(bot: FetchTarget):
desc='开启后当Minecraft基岩版商店更新时将会自动推送消息。', alias='mcbvrss')
async def mcbv_rss(bot: FetchTarget):
try:
version_file = os.path.abspath(f'{PrivateAssets.path}/mcbeversion.txt')
verlist = getfileversions(version_file)
verlist = get_stored_list(bot, 'mcbv_rss')
version = google_play_scraper('com.mojang.minecraftpe')['version']
if version not in verlist:
Logger.info(f'huh, we find bedrock {version}.')
await bot.post_message('mcbv_rss', '基岩版商店已更新' + version + '正式版。')
addversion = open(version_file, 'a', encoding='utf-8')
addversion.write('\n' + version)
addversion.close()
verlist.append(version)
update_stored_list(bot, 'mcbv_rss', verlist)
except Exception:
traceback.print_exc()
@ -136,23 +118,24 @@ async def mcbv_rss(bot: FetchTarget):
desc='开启后当Jira更新Java版时将会自动推送消息。', alias='mcvjirarss')
async def mcv_jira_rss(bot: FetchTarget):
try:
version_file = os.path.abspath(f'{PrivateAssets.path}/mcjira_Java.txt')
verlist = getfileversions(version_file)
verlist = get_stored_list(bot, 'mcv_jira_rss')
file = json.loads(await get_url('https://bugs.mojang.com/rest/api/2/project/10400/versions'))
releases = []
for v in file:
if not v['archived']:
releases.append(v['name'])
else:
if v['name'] not in verlist:
verlist.append(v['name'])
for release in releases:
if release not in verlist:
Logger.info(f'huh, we find {release}.')
verlist.append(release)
await bot.post_message('mcv_jira_rss',
f'Jira已更新Java版 {release}。'
f'\n（Jira上的信息仅作版本号预览用，不代表启动器已更新此版本）')
addversion = open(version_file, 'a', encoding='utf-8')
addversion.write('\n' + release)
addversion.close()
verlist.append(release)
update_stored_list(bot, 'mcv_jira_rss', verlist)
except Exception:
traceback.print_exc()
@ -164,23 +147,24 @@ async def mcv_jira_rss(bot: FetchTarget):
desc='开启后当Jira更新基岩版时将会自动推送消息。', alias='mcbvjirarss')
async def mcbv_jira_rss(bot: FetchTarget):
try:
version_file = os.path.abspath(f'{PrivateAssets.path}/mcjira_Bedrock.txt')
verlist = getfileversions(version_file)
verlist = get_stored_list(bot, 'mcbv_jira_rss')
file = json.loads(await get_url('https://bugs.mojang.com/rest/api/2/project/10200/versions'))
releases = []
for v in file:
if not v['archived']:
releases.append(v['name'])
else:
if v['name'] not in verlist:
verlist.append(v['name'])
for release in releases:
if release not in verlist:
Logger.info(f'huh, we find {release}.')
verlist.append(release)
await bot.post_message('mcbv_jira_rss',
f'Jira已更新基岩版 {release}。'
f'\n（Jira上的信息仅作版本号预览用，不代表商城已更新此版本）')
addversion = open(version_file, 'a', encoding='utf-8')
addversion.write('\n' + release)
addversion.close()
verlist.append(release)
update_stored_list(bot, 'mcbv_jira_rss', verlist)
except Exception:
traceback.print_exc()
@ -192,22 +176,23 @@ async def mcbv_jira_rss(bot: FetchTarget):
desc='开启后当Jira更新Dungeons版本时将会自动推送消息。', alias='mcdvjirarss')
async def mcdv_jira_rss(bot: FetchTarget):
try:
version_file = os.path.abspath(f'{PrivateAssets.path}/mcjira_Minecraft Dungeons.txt')
verlist = getfileversions(version_file)
verlist = get_stored_list(bot, 'mcdv_jira_rss')
file = json.loads(await get_url('https://bugs.mojang.com/rest/api/2/project/11901/versions'))
releases = []
for v in file:
if not v['archived']:
releases.append(v['name'])
else:
if v['name'] not in verlist:
verlist.append(v['name'])
for release in releases:
if release not in verlist:
Logger.info(f'huh, we find {release}.')
verlist.append(release)
await bot.post_message('mcdv_jira_rss',
f'Jira已更新Minecraft Dungeons {release}。'
f'\n（Jira上的信息仅作版本号预览用，不代表启动器/商城已更新此版本）')
addversion = open(version_file, 'a', encoding='utf-8')
addversion.write('\n' + release)
addversion.close()
verlist.append(release)
update_stored_list(bot, 'mcdv_jira_rss', verlist)
except Exception:
traceback.print_exc()

31
modules/minecraft_news/__init__.py
View file

@ -10,17 +10,7 @@ from config import Config
from core.component import on_schedule
from core.elements import FetchTarget, IntervalTrigger, PrivateAssets, Url
from core.logger import Logger
from core.utils import get_url
def getfileversions(path):
if not os.path.exists(path):
a = open(path, 'a', encoding='utf-8')
a.close()
w = open(path, 'r+', encoding='utf-8')
s = w.read().split('\n')
w.close()
return s
from core.utils import get_url, get_stored_list, update_stored_list
class Article:
@ -53,7 +43,6 @@ class Article:
recommend_modules=['feedback_news'], trigger=IntervalTrigger(seconds=60 if not Config('slower_schedule') else 180),
desc='开启后将会自动推送来自Minecraft官网的新闻。')
async def start_check_news(bot: FetchTarget):
file_ = os.path.abspath(f'{PrivateAssets.path}/mcnews.txt')
baseurl = 'https://www.minecraft.net'
url = quote(
f'https://www.minecraft.net/content/minecraft-net/_jcr_content.articles.grid?tileselection=auto&tagsPath={",".join(Article.random_tags())}&offset=0&pageSize={Article.count}')
@ -63,7 +52,7 @@ async def start_check_news(bot: FetchTarget):
return
getpage = await get_url(get)
if getpage:
alist = getfileversions(file_)
alist = get_stored_list(bot, 'mcnews')
o_json = json.loads(getpage)
o_nws = o_json['article_grid']
Article.count = o_json['article_count']
@ -78,9 +67,8 @@ async def start_check_news(bot: FetchTarget):
now = datetime.now()
if now - publish_date < timedelta(days=2):
await bot.post_message('minecraft_news', articletext)
addversion = open(file_, 'a', encoding='utf-8')
addversion.write('\n' + title)
addversion.close()
alist.append(title)
update_stored_list(bot, 'mcnews', alist)
@on_schedule('feedback_news', developers=['Dianliang233'], recommend_modules=['minecraft_news'],
@ -92,9 +80,7 @@ async def feedback_news(bot: FetchTarget):
'url': 'https://minecraftfeedback.zendesk.com/api/v2/help_center/en-us/sections/360001186971/articles?per_page=5'}]
for section in sections:
try:
name = section['name']
version_file = os.path.abspath(f'{PrivateAssets.path}/feedback_{name}.txt')
alist = getfileversions(version_file)
alist = get_stored_list(bot, 'mcfeedbacknews')
get = await get_url(section['url'])
res = json.loads(get)
articles = []
@ -105,12 +91,9 @@ async def feedback_news(bot: FetchTarget):
name = article['name']
link = article['html_url']
Logger.info(f'huh, we find {name}.')
alist.append(name)
await bot.post_message('feedback_news',
f'Minecraft Feedback 发布了新的文章：\n{name}\n{str(Url(link))}')
addversion = open(version_file, 'a', encoding='utf-8')
addversion.write('\n' + name)
addversion.close()
alist.append(name)
update_stored_list(bot, 'mcfeedbacknews', alist)
except Exception:
traceback.print_exc()
print(get)