use modified html2text
To remove link title
This commit is contained in:
yzhh
parent
eddfe06f9e
commit
808e599a4b
11 changed files with 1721 additions and 4 deletions
947
core/html2text/__init__.py
Normal file
947
core/html2text/__init__.py
Normal file
|
|
@ -0,0 +1,947 @@
|
|||
"""html2text: Turn HTML into equivalent Markdown-structured text."""
|
||||
|
||||
import html.entities
|
||||
import html.parser
|
||||
import re
|
||||
import urllib.parse as urlparse
|
||||
from textwrap import wrap
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
|
||||
from . import config
|
||||
from .elements import AnchorElement, ListElement
|
||||
from .typing import OutCallback
|
||||
from .utils import (
|
||||
dumb_css_parser,
|
||||
element_style,
|
||||
escape_md,
|
||||
escape_md_section,
|
||||
google_fixed_width_font,
|
||||
google_has_height,
|
||||
google_list_style,
|
||||
google_text_emphasis,
|
||||
hn,
|
||||
list_numbering_start,
|
||||
pad_tables_in_text,
|
||||
skipwrap,
|
||||
unifiable_n,
|
||||
)
|
||||
|
||||
__version__ = (2020, 1, 16)
|
||||
|
||||
|
||||
# TODO:
|
||||
# Support decoded entities with UNIFIABLE.
|
||||
|
||||
|
||||
class HTML2Text(html.parser.HTMLParser):
|
||||
def __init__(
|
||||
self,
|
||||
out: Optional[OutCallback] = None,
|
||||
baseurl: str = "",
|
||||
bodywidth: int = config.BODY_WIDTH,
|
||||
) -> None:
|
||||
"""
|
||||
Input parameters:
|
||||
out: possible custom replacement for self.outtextf (which
|
||||
appends lines of text).
|
||||
baseurl: base URL of the document we process
|
||||
"""
|
||||
super().__init__(convert_charrefs=False)
|
||||
|
||||
# Config options
|
||||
self.split_next_td = False
|
||||
self.td_count = 0
|
||||
self.table_start = False
|
||||
self.unicode_snob = config.UNICODE_SNOB # covered in cli
|
||||
self.escape_snob = config.ESCAPE_SNOB # covered in cli
|
||||
self.links_each_paragraph = config.LINKS_EACH_PARAGRAPH
|
||||
self.body_width = bodywidth # covered in cli
|
||||
self.skip_internal_links = config.SKIP_INTERNAL_LINKS # covered in cli
|
||||
self.inline_links = config.INLINE_LINKS # covered in cli
|
||||
self.protect_links = config.PROTECT_LINKS # covered in cli
|
||||
self.google_list_indent = config.GOOGLE_LIST_INDENT # covered in cli
|
||||
self.ignore_links = config.IGNORE_ANCHORS # covered in cli
|
||||
self.ignore_images = config.IGNORE_IMAGES # covered in cli
|
||||
self.images_as_html = config.IMAGES_AS_HTML # covered in cli
|
||||
self.images_to_alt = config.IMAGES_TO_ALT # covered in cli
|
||||
self.images_with_size = config.IMAGES_WITH_SIZE # covered in cli
|
||||
self.ignore_emphasis = config.IGNORE_EMPHASIS # covered in cli
|
||||
self.bypass_tables = config.BYPASS_TABLES # covered in cli
|
||||
self.ignore_tables = config.IGNORE_TABLES # covered in cli
|
||||
self.google_doc = False # covered in cli
|
||||
self.ul_item_mark = "*" # covered in cli
|
||||
self.emphasis_mark = "_" # covered in cli
|
||||
self.strong_mark = "**"
|
||||
self.single_line_break = config.SINGLE_LINE_BREAK # covered in cli
|
||||
self.use_automatic_links = config.USE_AUTOMATIC_LINKS # covered in cli
|
||||
self.hide_strikethrough = False # covered in cli
|
||||
self.mark_code = config.MARK_CODE
|
||||
self.wrap_list_items = config.WRAP_LIST_ITEMS # covered in cli
|
||||
self.wrap_links = config.WRAP_LINKS # covered in cli
|
||||
self.pad_tables = config.PAD_TABLES # covered in cli
|
||||
self.default_image_alt = config.DEFAULT_IMAGE_ALT # covered in cli
|
||||
self.tag_callback = None
|
||||
self.open_quote = config.OPEN_QUOTE # covered in cli
|
||||
self.close_quote = config.CLOSE_QUOTE # covered in cli
|
||||
|
||||
if out is None:
|
||||
self.out = self.outtextf
|
||||
else:
|
||||
self.out = out
|
||||
|
||||
# empty list to store output characters before they are "joined"
|
||||
self.outtextlist = [] # type: List[str]
|
||||
|
||||
self.quiet = 0
|
||||
self.p_p = 0 # number of newline character to print before next output
|
||||
self.outcount = 0
|
||||
self.start = True
|
||||
self.space = False
|
||||
self.a = [] # type: List[AnchorElement]
|
||||
self.astack = [] # type: List[Optional[Dict[str, Optional[str]]]]
|
||||
self.maybe_automatic_link = None # type: Optional[str]
|
||||
self.empty_link = False
|
||||
self.absolute_url_matcher = re.compile(r"^[a-zA-Z+]+://")
|
||||
self.acount = 0
|
||||
self.list = [] # type: List[ListElement]
|
||||
self.blockquote = 0
|
||||
self.pre = False
|
||||
self.startpre = False
|
||||
self.code = False
|
||||
self.quote = False
|
||||
self.br_toggle = ""
|
||||
self.lastWasNL = False
|
||||
self.lastWasList = False
|
||||
self.style = 0
|
||||
self.style_def = {} # type: Dict[str, Dict[str, str]]
|
||||
self.tag_stack = (
|
||||
[]
|
||||
) # type: List[Tuple[str, Dict[str, Optional[str]], Dict[str, str]]]
|
||||
self.emphasis = 0
|
||||
self.drop_white_space = 0
|
||||
self.inheader = False
|
||||
# Current abbreviation definition
|
||||
self.abbr_title = None # type: Optional[str]
|
||||
# Last inner HTML (for abbr being defined)
|
||||
self.abbr_data = None # type: Optional[str]
|
||||
# Stack of abbreviations to write later
|
||||
self.abbr_list = {} # type: Dict[str, str]
|
||||
self.baseurl = baseurl
|
||||
self.stressed = False
|
||||
self.preceding_stressed = False
|
||||
self.preceding_data = ""
|
||||
self.current_tag = ""
|
||||
|
||||
config.UNIFIABLE["nbsp"] = " _place_holder;"
|
||||
|
||||
def feed(self, data: str) -> None:
|
||||
data = data.replace("</' + 'script>", "</ignore>")
|
||||
super().feed(data)
|
||||
|
||||
def handle(self, data: str) -> str:
|
||||
self.feed(data)
|
||||
self.feed("")
|
||||
markdown = self.optwrap(self.finish())
|
||||
if self.pad_tables:
|
||||
return pad_tables_in_text(markdown)
|
||||
else:
|
||||
return markdown
|
||||
|
||||
def outtextf(self, s: str) -> None:
|
||||
self.outtextlist.append(s)
|
||||
if s:
|
||||
self.lastWasNL = s[-1] == "\n"
|
||||
|
||||
def finish(self) -> str:
|
||||
self.close()
|
||||
|
||||
self.pbr()
|
||||
self.o("", force="end")
|
||||
|
||||
outtext = "".join(self.outtextlist)
|
||||
|
||||
if self.unicode_snob:
|
||||
nbsp = html.entities.html5["nbsp;"]
|
||||
else:
|
||||
nbsp = " "
|
||||
outtext = outtext.replace(" _place_holder;", nbsp)
|
||||
|
||||
# Clear self.outtextlist to avoid memory leak of its content to
|
||||
# the next handling.
|
||||
self.outtextlist = []
|
||||
|
||||
return outtext
|
||||
|
||||
def handle_charref(self, c: str) -> None:
|
||||
self.handle_data(self.charref(c), True)
|
||||
|
||||
def handle_entityref(self, c: str) -> None:
|
||||
ref = self.entityref(c)
|
||||
|
||||
# ref may be an empty string (e.g. for ‎/‏ markers that should
|
||||
# not contribute to the final output).
|
||||
# self.handle_data cannot handle a zero-length string right after a
|
||||
# stressed tag or mid-text within a stressed tag (text get split and
|
||||
# self.stressed/self.preceding_stressed gets switched after the first
|
||||
# part of that text).
|
||||
if ref:
|
||||
self.handle_data(ref, True)
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None:
|
||||
self.handle_tag(tag, dict(attrs), start=True)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
self.handle_tag(tag, {}, start=False)
|
||||
|
||||
def previousIndex(self, attrs: Dict[str, Optional[str]]) -> Optional[int]:
|
||||
"""
|
||||
:type attrs: dict
|
||||
|
||||
:returns: The index of certain set of attributes (of a link) in the
|
||||
self.a list. If the set of attributes is not found, returns None
|
||||
:rtype: int
|
||||
"""
|
||||
if "href" not in attrs:
|
||||
return None
|
||||
|
||||
match = False
|
||||
for i, a in enumerate(self.a):
|
||||
if "href" in a.attrs and a.attrs["href"] == attrs["href"]:
|
||||
if "title" in a.attrs or "title" in attrs:
|
||||
if (
|
||||
"title" in a.attrs
|
||||
and "title" in attrs
|
||||
and a.attrs["title"] == attrs["title"]
|
||||
):
|
||||
match = True
|
||||
else:
|
||||
match = True
|
||||
|
||||
if match:
|
||||
return i
|
||||
return None
|
||||
|
||||
def handle_emphasis(
|
||||
self, start: bool, tag_style: Dict[str, str], parent_style: Dict[str, str]
|
||||
) -> None:
|
||||
"""
|
||||
Handles various text emphases
|
||||
"""
|
||||
tag_emphasis = google_text_emphasis(tag_style)
|
||||
parent_emphasis = google_text_emphasis(parent_style)
|
||||
|
||||
# handle Google's text emphasis
|
||||
strikethrough = "line-through" in tag_emphasis and self.hide_strikethrough
|
||||
|
||||
# google and others may mark a font's weight as `bold` or `700`
|
||||
bold = False
|
||||
for bold_marker in config.BOLD_TEXT_STYLE_VALUES:
|
||||
bold = bold_marker in tag_emphasis and bold_marker not in parent_emphasis
|
||||
if bold:
|
||||
break
|
||||
|
||||
italic = "italic" in tag_emphasis and "italic" not in parent_emphasis
|
||||
fixed = (
|
||||
google_fixed_width_font(tag_style)
|
||||
and not google_fixed_width_font(parent_style)
|
||||
and not self.pre
|
||||
)
|
||||
|
||||
if start:
|
||||
# crossed-out text must be handled before other attributes
|
||||
# in order not to output qualifiers unnecessarily
|
||||
if bold or italic or fixed:
|
||||
self.emphasis += 1
|
||||
if strikethrough:
|
||||
self.quiet += 1
|
||||
if italic:
|
||||
self.o(self.emphasis_mark)
|
||||
self.drop_white_space += 1
|
||||
if bold:
|
||||
self.o(self.strong_mark)
|
||||
self.drop_white_space += 1
|
||||
if fixed:
|
||||
self.o("`")
|
||||
self.drop_white_space += 1
|
||||
self.code = True
|
||||
else:
|
||||
if bold or italic or fixed:
|
||||
# there must not be whitespace before closing emphasis mark
|
||||
self.emphasis -= 1
|
||||
self.space = False
|
||||
if fixed:
|
||||
if self.drop_white_space:
|
||||
# empty emphasis, drop it
|
||||
self.drop_white_space -= 1
|
||||
else:
|
||||
self.o("`")
|
||||
self.code = False
|
||||
if bold:
|
||||
if self.drop_white_space:
|
||||
# empty emphasis, drop it
|
||||
self.drop_white_space -= 1
|
||||
else:
|
||||
self.o(self.strong_mark)
|
||||
if italic:
|
||||
if self.drop_white_space:
|
||||
# empty emphasis, drop it
|
||||
self.drop_white_space -= 1
|
||||
else:
|
||||
self.o(self.emphasis_mark)
|
||||
# space is only allowed after *all* emphasis marks
|
||||
if (bold or italic) and not self.emphasis:
|
||||
self.o(" ")
|
||||
if strikethrough:
|
||||
self.quiet -= 1
|
||||
|
||||
def handle_tag(
|
||||
self, tag: str, attrs: Dict[str, Optional[str]], start: bool
|
||||
) -> None:
|
||||
self.current_tag = tag
|
||||
|
||||
if self.tag_callback is not None:
|
||||
if self.tag_callback(self, tag, attrs, start) is True:
|
||||
return
|
||||
|
||||
# first thing inside the anchor tag is another tag
|
||||
# that produces some output
|
||||
if (
|
||||
start
|
||||
and self.maybe_automatic_link is not None
|
||||
and tag not in ["p", "div", "style", "dl", "dt"]
|
||||
and (tag != "img" or self.ignore_images)
|
||||
):
|
||||
self.o("[")
|
||||
self.maybe_automatic_link = None
|
||||
self.empty_link = False
|
||||
|
||||
if self.google_doc:
|
||||
# the attrs parameter is empty for a closing tag. in addition, we
|
||||
# need the attributes of the parent nodes in order to get a
|
||||
# complete style description for the current element. we assume
|
||||
# that google docs export well formed html.
|
||||
parent_style = {} # type: Dict[str, str]
|
||||
if start:
|
||||
if self.tag_stack:
|
||||
parent_style = self.tag_stack[-1][2]
|
||||
tag_style = element_style(attrs, self.style_def, parent_style)
|
||||
self.tag_stack.append((tag, attrs, tag_style))
|
||||
else:
|
||||
dummy, attrs, tag_style = (
|
||||
self.tag_stack.pop() if self.tag_stack else (None, {}, {})
|
||||
)
|
||||
if self.tag_stack:
|
||||
parent_style = self.tag_stack[-1][2]
|
||||
|
||||
if hn(tag):
|
||||
self.p()
|
||||
if start:
|
||||
self.inheader = True
|
||||
self.o(hn(tag) * "#" + " ")
|
||||
else:
|
||||
self.inheader = False
|
||||
return # prevent redundant emphasis marks on headers
|
||||
|
||||
if tag in ["p", "div"]:
|
||||
if self.google_doc:
|
||||
if start and google_has_height(tag_style):
|
||||
self.p()
|
||||
else:
|
||||
self.soft_br()
|
||||
elif self.astack and tag == "div":
|
||||
pass
|
||||
else:
|
||||
self.p()
|
||||
|
||||
if tag == "br" and start:
|
||||
if self.blockquote > 0:
|
||||
self.o(" \n> ")
|
||||
else:
|
||||
self.o(" \n")
|
||||
|
||||
if tag == "hr" and start:
|
||||
self.p()
|
||||
self.o("* * *")
|
||||
self.p()
|
||||
|
||||
if tag in ["head", "style", "script"]:
|
||||
if start:
|
||||
self.quiet += 1
|
||||
else:
|
||||
self.quiet -= 1
|
||||
|
||||
if tag == "style":
|
||||
if start:
|
||||
self.style += 1
|
||||
else:
|
||||
self.style -= 1
|
||||
|
||||
if tag in ["body"]:
|
||||
self.quiet = 0 # sites like 9rules.com never close <head>
|
||||
|
||||
if tag == "blockquote":
|
||||
if start:
|
||||
self.p()
|
||||
self.o("> ", force=True)
|
||||
self.start = True
|
||||
self.blockquote += 1
|
||||
else:
|
||||
self.blockquote -= 1
|
||||
self.p()
|
||||
|
||||
def no_preceding_space(self: HTML2Text) -> bool:
|
||||
return bool(
|
||||
self.preceding_data and re.match(r"[^\s]", self.preceding_data[-1])
|
||||
)
|
||||
|
||||
if tag in ["em", "i", "u"] and not self.ignore_emphasis:
|
||||
if start and no_preceding_space(self):
|
||||
emphasis = " " + self.emphasis_mark
|
||||
else:
|
||||
emphasis = self.emphasis_mark
|
||||
|
||||
self.o(emphasis)
|
||||
if start:
|
||||
self.stressed = True
|
||||
|
||||
if tag in ["strong", "b"] and not self.ignore_emphasis:
|
||||
if start and no_preceding_space(self):
|
||||
strong = " " + self.strong_mark
|
||||
else:
|
||||
strong = self.strong_mark
|
||||
|
||||
self.o(strong)
|
||||
if start:
|
||||
self.stressed = True
|
||||
|
||||
if tag in ["del", "strike", "s"]:
|
||||
if start and no_preceding_space(self):
|
||||
strike = " ~~"
|
||||
else:
|
||||
strike = "~~"
|
||||
|
||||
self.o(strike)
|
||||
if start:
|
||||
self.stressed = True
|
||||
|
||||
if self.google_doc:
|
||||
if not self.inheader:
|
||||
# handle some font attributes, but leave headers clean
|
||||
self.handle_emphasis(start, tag_style, parent_style)
|
||||
|
||||
if tag in ["kbd", "code", "tt"] and not self.pre:
|
||||
self.o("`") # TODO: `` `this` ``
|
||||
self.code = not self.code
|
||||
|
||||
if tag == "abbr":
|
||||
if start:
|
||||
self.abbr_title = None
|
||||
self.abbr_data = ""
|
||||
if "title" in attrs:
|
||||
self.abbr_title = attrs["title"]
|
||||
else:
|
||||
if self.abbr_title is not None:
|
||||
assert self.abbr_data is not None
|
||||
self.abbr_list[self.abbr_data] = self.abbr_title
|
||||
self.abbr_title = None
|
||||
self.abbr_data = None
|
||||
|
||||
if tag == "q":
|
||||
if not self.quote:
|
||||
self.o(self.open_quote)
|
||||
else:
|
||||
self.o(self.close_quote)
|
||||
self.quote = not self.quote
|
||||
|
||||
def link_url(self: HTML2Text, link: str, title: str = "") -> None:
|
||||
url = urlparse.urljoin(self.baseurl, link)
|
||||
# title = ' "{}"'.format(title) if title.strip() else ""
|
||||
self.o("]({url})".format(url=escape_md(url)))
|
||||
|
||||
if tag == "a" and not self.ignore_links:
|
||||
if start:
|
||||
if (
|
||||
"href" in attrs
|
||||
and attrs["href"] is not None
|
||||
and not (self.skip_internal_links and attrs["href"].startswith("#"))
|
||||
):
|
||||
self.astack.append(attrs)
|
||||
self.maybe_automatic_link = attrs["href"]
|
||||
self.empty_link = True
|
||||
if self.protect_links:
|
||||
attrs["href"] = "<" + attrs["href"] + ">"
|
||||
else:
|
||||
self.astack.append(None)
|
||||
else:
|
||||
if self.astack:
|
||||
a = self.astack.pop()
|
||||
if self.maybe_automatic_link and not self.empty_link:
|
||||
self.maybe_automatic_link = None
|
||||
elif a:
|
||||
assert a["href"] is not None
|
||||
if self.empty_link:
|
||||
self.o("[")
|
||||
self.empty_link = False
|
||||
self.maybe_automatic_link = None
|
||||
if self.inline_links:
|
||||
title = a.get("title") or ""
|
||||
title = escape_md(title)
|
||||
link_url(self, a["href"], title)
|
||||
else:
|
||||
i = self.previousIndex(a)
|
||||
if i is not None:
|
||||
a_props = self.a[i]
|
||||
else:
|
||||
self.acount += 1
|
||||
a_props = AnchorElement(a, self.acount, self.outcount)
|
||||
self.a.append(a_props)
|
||||
self.o("][" + str(a_props.count) + "]")
|
||||
|
||||
if tag == "img" and start and not self.ignore_images:
|
||||
if "src" in attrs:
|
||||
assert attrs["src"] is not None
|
||||
if not self.images_to_alt:
|
||||
attrs["href"] = attrs["src"]
|
||||
alt = attrs.get("alt") or self.default_image_alt
|
||||
|
||||
# If we have images_with_size, write raw html including width,
|
||||
# height, and alt attributes
|
||||
if self.images_as_html or (
|
||||
self.images_with_size and ("width" in attrs or "height" in attrs)
|
||||
):
|
||||
self.o("<img src='" + attrs["src"] + "' ")
|
||||
if "width" in attrs:
|
||||
assert attrs["width"] is not None
|
||||
self.o("width='" + attrs["width"] + "' ")
|
||||
if "height" in attrs:
|
||||
assert attrs["height"] is not None
|
||||
self.o("height='" + attrs["height"] + "' ")
|
||||
if alt:
|
||||
self.o("alt='" + alt + "' ")
|
||||
self.o("/>")
|
||||
return
|
||||
|
||||
# If we have a link to create, output the start
|
||||
if self.maybe_automatic_link is not None:
|
||||
href = self.maybe_automatic_link
|
||||
if (
|
||||
self.images_to_alt
|
||||
and escape_md(alt) == href
|
||||
and self.absolute_url_matcher.match(href)
|
||||
):
|
||||
self.o("<" + escape_md(alt) + ">")
|
||||
self.empty_link = False
|
||||
return
|
||||
else:
|
||||
self.o("[")
|
||||
self.maybe_automatic_link = None
|
||||
self.empty_link = False
|
||||
|
||||
# If we have images_to_alt, we discard the image itself,
|
||||
# considering only the alt text.
|
||||
if self.images_to_alt:
|
||||
self.o(escape_md(alt))
|
||||
else:
|
||||
self.o("![" + escape_md(alt) + "]")
|
||||
if self.inline_links:
|
||||
href = attrs.get("href") or ""
|
||||
self.o(
|
||||
"(" + escape_md(urlparse.urljoin(self.baseurl, href)) + ")"
|
||||
)
|
||||
else:
|
||||
i = self.previousIndex(attrs)
|
||||
if i is not None:
|
||||
a_props = self.a[i]
|
||||
else:
|
||||
self.acount += 1
|
||||
a_props = AnchorElement(attrs, self.acount, self.outcount)
|
||||
self.a.append(a_props)
|
||||
self.o("[" + str(a_props.count) + "]")
|
||||
|
||||
if tag == "dl" and start:
|
||||
self.p()
|
||||
if tag == "dt" and not start:
|
||||
self.pbr()
|
||||
if tag == "dd" and start:
|
||||
self.o(" ")
|
||||
if tag == "dd" and not start:
|
||||
self.pbr()
|
||||
|
||||
if tag in ["ol", "ul"]:
|
||||
# Google Docs create sub lists as top level lists
|
||||
if not self.list and not self.lastWasList:
|
||||
self.p()
|
||||
if start:
|
||||
if self.google_doc:
|
||||
list_style = google_list_style(tag_style)
|
||||
else:
|
||||
list_style = tag
|
||||
numbering_start = list_numbering_start(attrs)
|
||||
self.list.append(ListElement(list_style, numbering_start))
|
||||
else:
|
||||
if self.list:
|
||||
self.list.pop()
|
||||
if not self.google_doc and not self.list:
|
||||
self.o("\n")
|
||||
self.lastWasList = True
|
||||
else:
|
||||
self.lastWasList = False
|
||||
|
||||
if tag == "li":
|
||||
self.pbr()
|
||||
if start:
|
||||
if self.list:
|
||||
li = self.list[-1]
|
||||
else:
|
||||
li = ListElement("ul", 0)
|
||||
if self.google_doc:
|
||||
nest_count = self.google_nest_count(tag_style)
|
||||
else:
|
||||
nest_count = len(self.list)
|
||||
# TODO: line up <ol><li>s > 9 correctly.
|
||||
self.o(" " * nest_count)
|
||||
if li.name == "ul":
|
||||
self.o(self.ul_item_mark + " ")
|
||||
elif li.name == "ol":
|
||||
li.num += 1
|
||||
self.o(str(li.num) + ". ")
|
||||
self.start = True
|
||||
|
||||
if tag in ["table", "tr", "td", "th"]:
|
||||
if self.ignore_tables:
|
||||
if tag == "tr":
|
||||
if start:
|
||||
pass
|
||||
else:
|
||||
self.soft_br()
|
||||
else:
|
||||
pass
|
||||
|
||||
elif self.bypass_tables:
|
||||
if start:
|
||||
self.soft_br()
|
||||
if tag in ["td", "th"]:
|
||||
if start:
|
||||
self.o("<{}>\n\n".format(tag))
|
||||
else:
|
||||
self.o("\n</{}>".format(tag))
|
||||
else:
|
||||
if start:
|
||||
self.o("<{}>".format(tag))
|
||||
else:
|
||||
self.o("</{}>".format(tag))
|
||||
|
||||
else:
|
||||
if tag == "table":
|
||||
if start:
|
||||
self.table_start = True
|
||||
if self.pad_tables:
|
||||
self.o("<" + config.TABLE_MARKER_FOR_PAD + ">")
|
||||
self.o(" \n")
|
||||
else:
|
||||
if self.pad_tables:
|
||||
self.o("</" + config.TABLE_MARKER_FOR_PAD + ">")
|
||||
self.o(" \n")
|
||||
if tag in ["td", "th"] and start:
|
||||
if self.split_next_td:
|
||||
self.o("| ")
|
||||
self.split_next_td = True
|
||||
|
||||
if tag == "tr" and start:
|
||||
self.td_count = 0
|
||||
if tag == "tr" and not start:
|
||||
self.split_next_td = False
|
||||
self.soft_br()
|
||||
if tag == "tr" and not start and self.table_start:
|
||||
# Underline table header
|
||||
self.o("|".join(["---"] * self.td_count))
|
||||
self.soft_br()
|
||||
self.table_start = False
|
||||
if tag in ["td", "th"] and start:
|
||||
self.td_count += 1
|
||||
|
||||
if tag == "pre":
|
||||
if start:
|
||||
self.startpre = True
|
||||
self.pre = True
|
||||
else:
|
||||
self.pre = False
|
||||
if self.mark_code:
|
||||
self.out("\n[/code]")
|
||||
self.p()
|
||||
|
||||
# TODO: Add docstring for these one letter functions
|
||||
def pbr(self) -> None:
|
||||
"Pretty print has a line break"
|
||||
if self.p_p == 0:
|
||||
self.p_p = 1
|
||||
|
||||
def p(self) -> None:
|
||||
"Set pretty print to 1 or 2 lines"
|
||||
self.p_p = 1 if self.single_line_break else 2
|
||||
|
||||
def soft_br(self) -> None:
|
||||
"Soft breaks"
|
||||
self.pbr()
|
||||
self.br_toggle = " "
|
||||
|
||||
def o(
|
||||
self, data: str, puredata: bool = False, force: Union[bool, str] = False
|
||||
) -> None:
|
||||
"""
|
||||
Deal with indentation and whitespace
|
||||
"""
|
||||
if self.abbr_data is not None:
|
||||
self.abbr_data += data
|
||||
|
||||
if not self.quiet:
|
||||
if self.google_doc:
|
||||
# prevent white space immediately after 'begin emphasis'
|
||||
# marks ('**' and '_')
|
||||
lstripped_data = data.lstrip()
|
||||
if self.drop_white_space and not (self.pre or self.code):
|
||||
data = lstripped_data
|
||||
if lstripped_data != "":
|
||||
self.drop_white_space = 0
|
||||
|
||||
if puredata and not self.pre:
|
||||
# This is a very dangerous call ... it could mess up
|
||||
# all handling of when not handled properly
|
||||
# (see entityref)
|
||||
data = re.sub(r"\s+", r" ", data)
|
||||
if data and data[0] == " ":
|
||||
self.space = True
|
||||
data = data[1:]
|
||||
if not data and not force:
|
||||
return
|
||||
|
||||
if self.startpre:
|
||||
# self.out(" :") #TODO: not output when already one there
|
||||
if not data.startswith("\n") and not data.startswith("\r\n"):
|
||||
# <pre>stuff...
|
||||
data = "\n" + data
|
||||
if self.mark_code:
|
||||
self.out("\n[code]")
|
||||
self.p_p = 0
|
||||
|
||||
bq = ">" * self.blockquote
|
||||
if not (force and data and data[0] == ">") and self.blockquote:
|
||||
bq += " "
|
||||
|
||||
if self.pre:
|
||||
if not self.list:
|
||||
bq += " "
|
||||
# else: list content is already partially indented
|
||||
bq += " " * len(self.list)
|
||||
data = data.replace("\n", "\n" + bq)
|
||||
|
||||
if self.startpre:
|
||||
self.startpre = False
|
||||
if self.list:
|
||||
# use existing initial indentation
|
||||
data = data.lstrip("\n")
|
||||
|
||||
if self.start:
|
||||
self.space = False
|
||||
self.p_p = 0
|
||||
self.start = False
|
||||
|
||||
if force == "end":
|
||||
# It's the end.
|
||||
self.p_p = 0
|
||||
self.out("\n")
|
||||
self.space = False
|
||||
|
||||
if self.p_p:
|
||||
self.out((self.br_toggle + "\n" + bq) * self.p_p)
|
||||
self.space = False
|
||||
self.br_toggle = ""
|
||||
|
||||
if self.space:
|
||||
if not self.lastWasNL:
|
||||
self.out(" ")
|
||||
self.space = False
|
||||
|
||||
if self.a and (
|
||||
(self.p_p == 2 and self.links_each_paragraph) or force == "end"
|
||||
):
|
||||
if force == "end":
|
||||
self.out("\n")
|
||||
|
||||
newa = []
|
||||
for link in self.a:
|
||||
if self.outcount > link.outcount:
|
||||
self.out(
|
||||
" ["
|
||||
+ str(link.count)
|
||||
+ "]: "
|
||||
+ urlparse.urljoin(self.baseurl, link.attrs["href"])
|
||||
)
|
||||
if "title" in link.attrs:
|
||||
assert link.attrs["title"] is not None
|
||||
self.out(" (" + link.attrs["title"] + ")")
|
||||
self.out("\n")
|
||||
else:
|
||||
newa.append(link)
|
||||
|
||||
# Don't need an extra line when nothing was done.
|
||||
if self.a != newa:
|
||||
self.out("\n")
|
||||
|
||||
self.a = newa
|
||||
|
||||
if self.abbr_list and force == "end":
|
||||
for abbr, definition in self.abbr_list.items():
|
||||
self.out(" *[" + abbr + "]: " + definition + "\n")
|
||||
|
||||
self.p_p = 0
|
||||
self.out(data)
|
||||
self.outcount += 1
|
||||
|
||||
def handle_data(self, data: str, entity_char: bool = False) -> None:
|
||||
if not data:
|
||||
# Data may be empty for some HTML entities. For example,
|
||||
# LEFT-TO-RIGHT MARK.
|
||||
return
|
||||
|
||||
if self.stressed:
|
||||
data = data.strip()
|
||||
self.stressed = False
|
||||
self.preceding_stressed = True
|
||||
elif self.preceding_stressed:
|
||||
if (
|
||||
re.match(r"[^\s.!?]", data[0])
|
||||
and not hn(self.current_tag)
|
||||
and self.current_tag not in ["a", "code", "pre"]
|
||||
):
|
||||
# should match a letter or common punctuation
|
||||
data = " " + data
|
||||
self.preceding_stressed = False
|
||||
|
||||
if self.style:
|
||||
self.style_def.update(dumb_css_parser(data))
|
||||
|
||||
if self.maybe_automatic_link is not None:
|
||||
href = self.maybe_automatic_link
|
||||
if (
|
||||
href == data
|
||||
and self.absolute_url_matcher.match(href)
|
||||
and self.use_automatic_links
|
||||
):
|
||||
self.o("<" + data + ">")
|
||||
self.empty_link = False
|
||||
return
|
||||
else:
|
||||
self.o("[")
|
||||
self.maybe_automatic_link = None
|
||||
self.empty_link = False
|
||||
|
||||
if not self.code and not self.pre and not entity_char:
|
||||
data = escape_md_section(data, snob=self.escape_snob)
|
||||
self.preceding_data = data
|
||||
self.o(data, puredata=True)
|
||||
|
||||
def charref(self, name: str) -> str:
|
||||
if name[0] in ["x", "X"]:
|
||||
c = int(name[1:], 16)
|
||||
else:
|
||||
c = int(name)
|
||||
|
||||
if not self.unicode_snob and c in unifiable_n:
|
||||
return unifiable_n[c]
|
||||
else:
|
||||
try:
|
||||
return chr(c)
|
||||
except ValueError: # invalid unicode
|
||||
return ""
|
||||
|
||||
def entityref(self, c: str) -> str:
|
||||
if not self.unicode_snob and c in config.UNIFIABLE:
|
||||
return config.UNIFIABLE[c]
|
||||
try:
|
||||
ch = html.entities.html5[c + ";"]
|
||||
except KeyError:
|
||||
return "&" + c + ";"
|
||||
return config.UNIFIABLE[c] if c == "nbsp" else ch
|
||||
|
||||
def google_nest_count(self, style: Dict[str, str]) -> int:
|
||||
"""
|
||||
Calculate the nesting count of google doc lists
|
||||
|
||||
:type style: dict
|
||||
|
||||
:rtype: int
|
||||
"""
|
||||
nest_count = 0
|
||||
if "margin-left" in style:
|
||||
nest_count = int(style["margin-left"][:-2]) // self.google_list_indent
|
||||
|
||||
return nest_count
|
||||
|
||||
def optwrap(self, text: str) -> str:
|
||||
"""
|
||||
Wrap all paragraphs in the provided text.
|
||||
|
||||
:type text: str
|
||||
|
||||
:rtype: str
|
||||
"""
|
||||
if not self.body_width:
|
||||
return text
|
||||
|
||||
result = ""
|
||||
newlines = 0
|
||||
# I cannot think of a better solution for now.
|
||||
# To avoid the non-wrap behaviour for entire paras
|
||||
# because of the presence of a link in it
|
||||
if not self.wrap_links:
|
||||
self.inline_links = False
|
||||
for para in text.split("\n"):
|
||||
if len(para) > 0:
|
||||
if not skipwrap(para, self.wrap_links, self.wrap_list_items):
|
||||
indent = ""
|
||||
if para.startswith(" " + self.ul_item_mark):
|
||||
# list item continuation: add a double indent to the
|
||||
# new lines
|
||||
indent = " "
|
||||
elif para.startswith("> "):
|
||||
# blockquote continuation: add the greater than symbol
|
||||
# to the new lines
|
||||
indent = "> "
|
||||
wrapped = wrap(
|
||||
para,
|
||||
self.body_width,
|
||||
break_long_words=False,
|
||||
subsequent_indent=indent,
|
||||
)
|
||||
result += "\n".join(wrapped)
|
||||
if para.endswith(" "):
|
||||
result += " \n"
|
||||
newlines = 1
|
||||
elif indent:
|
||||
result += "\n"
|
||||
newlines = 1
|
||||
else:
|
||||
result += "\n\n"
|
||||
newlines = 2
|
||||
else:
|
||||
# Warning for the tempted!!!
|
||||
# Be aware that obvious replacement of this with
|
||||
# line.isspace()
|
||||
# DOES NOT work! Explanations are welcome.
|
||||
if not config.RE_SPACE.match(para):
|
||||
result += para + "\n"
|
||||
newlines = 1
|
||||
else:
|
||||
if newlines < 2:
|
||||
result += "\n"
|
||||
newlines += 1
|
||||
return result
|
||||
|
||||
|
||||
def html2text(html: str, baseurl: str = "", bodywidth: Optional[int] = None) -> str:
|
||||
if bodywidth is None:
|
||||
bodywidth = config.BODY_WIDTH
|
||||
h = HTML2Text(baseurl=baseurl, bodywidth=bodywidth)
|
||||
|
||||
return h.handle(html)
|
||||
3
core/html2text/__main__.py
Normal file
3
core/html2text/__main__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from .cli import main
|
||||
|
||||
main()
|
||||
306
core/html2text/cli.py
Normal file
306
core/html2text/cli.py
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
import argparse
|
||||
import sys
|
||||
|
||||
from . import HTML2Text, __version__, config
|
||||
|
||||
|
||||
def main() -> None:
|
||||
baseurl = ""
|
||||
|
||||
class bcolors:
|
||||
HEADER = "\033[95m"
|
||||
OKBLUE = "\033[94m"
|
||||
OKGREEN = "\033[92m"
|
||||
WARNING = "\033[93m"
|
||||
FAIL = "\033[91m"
|
||||
ENDC = "\033[0m"
|
||||
BOLD = "\033[1m"
|
||||
UNDERLINE = "\033[4m"
|
||||
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument(
|
||||
"--default-image-alt",
|
||||
dest="default_image_alt",
|
||||
default=config.DEFAULT_IMAGE_ALT,
|
||||
help="The default alt string for images with missing ones",
|
||||
)
|
||||
p.add_argument(
|
||||
"--pad-tables",
|
||||
dest="pad_tables",
|
||||
action="store_true",
|
||||
default=config.PAD_TABLES,
|
||||
help="pad the cells to equal column width in tables",
|
||||
)
|
||||
p.add_argument(
|
||||
"--no-wrap-links",
|
||||
dest="wrap_links",
|
||||
action="store_false",
|
||||
default=config.WRAP_LINKS,
|
||||
help="don't wrap links during conversion",
|
||||
)
|
||||
p.add_argument(
|
||||
"--wrap-list-items",
|
||||
dest="wrap_list_items",
|
||||
action="store_true",
|
||||
default=config.WRAP_LIST_ITEMS,
|
||||
help="wrap list items during conversion",
|
||||
)
|
||||
p.add_argument(
|
||||
"--ignore-emphasis",
|
||||
dest="ignore_emphasis",
|
||||
action="store_true",
|
||||
default=config.IGNORE_EMPHASIS,
|
||||
help="don't include any formatting for emphasis",
|
||||
)
|
||||
p.add_argument(
|
||||
"--reference-links",
|
||||
dest="inline_links",
|
||||
action="store_false",
|
||||
default=config.INLINE_LINKS,
|
||||
help="use reference style links instead of inline links",
|
||||
)
|
||||
p.add_argument(
|
||||
"--ignore-links",
|
||||
dest="ignore_links",
|
||||
action="store_true",
|
||||
default=config.IGNORE_ANCHORS,
|
||||
help="don't include any formatting for links",
|
||||
)
|
||||
p.add_argument(
|
||||
"--protect-links",
|
||||
dest="protect_links",
|
||||
action="store_true",
|
||||
default=config.PROTECT_LINKS,
|
||||
help="protect links from line breaks surrounding them with angle brackets",
|
||||
)
|
||||
p.add_argument(
|
||||
"--ignore-images",
|
||||
dest="ignore_images",
|
||||
action="store_true",
|
||||
default=config.IGNORE_IMAGES,
|
||||
help="don't include any formatting for images",
|
||||
)
|
||||
p.add_argument(
|
||||
"--images-as-html",
|
||||
dest="images_as_html",
|
||||
action="store_true",
|
||||
default=config.IMAGES_AS_HTML,
|
||||
help=(
|
||||
"Always write image tags as raw html; preserves `height`, `width` and "
|
||||
"`alt` if possible."
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--images-to-alt",
|
||||
dest="images_to_alt",
|
||||
action="store_true",
|
||||
default=config.IMAGES_TO_ALT,
|
||||
help="Discard image data, only keep alt text",
|
||||
)
|
||||
p.add_argument(
|
||||
"--images-with-size",
|
||||
dest="images_with_size",
|
||||
action="store_true",
|
||||
default=config.IMAGES_WITH_SIZE,
|
||||
help=(
|
||||
"Write image tags with height and width attrs as raw html to retain "
|
||||
"dimensions"
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"-g",
|
||||
"--google-doc",
|
||||
action="store_true",
|
||||
dest="google_doc",
|
||||
default=False,
|
||||
help="convert an html-exported Google Document",
|
||||
)
|
||||
p.add_argument(
|
||||
"-d",
|
||||
"--dash-unordered-list",
|
||||
action="store_true",
|
||||
dest="ul_style_dash",
|
||||
default=False,
|
||||
help="use a dash rather than a star for unordered list items",
|
||||
)
|
||||
p.add_argument(
|
||||
"-e",
|
||||
"--asterisk-emphasis",
|
||||
action="store_true",
|
||||
dest="em_style_asterisk",
|
||||
default=False,
|
||||
help="use an asterisk rather than an underscore for emphasized text",
|
||||
)
|
||||
p.add_argument(
|
||||
"-b",
|
||||
"--body-width",
|
||||
dest="body_width",
|
||||
type=int,
|
||||
default=config.BODY_WIDTH,
|
||||
help="number of characters per output line, 0 for no wrap",
|
||||
)
|
||||
p.add_argument(
|
||||
"-i",
|
||||
"--google-list-indent",
|
||||
dest="list_indent",
|
||||
type=int,
|
||||
default=config.GOOGLE_LIST_INDENT,
|
||||
help="number of pixels Google indents nested lists",
|
||||
)
|
||||
p.add_argument(
|
||||
"-s",
|
||||
"--hide-strikethrough",
|
||||
action="store_true",
|
||||
dest="hide_strikethrough",
|
||||
default=False,
|
||||
help="hide strike-through text. only relevant when -g is " "specified as well",
|
||||
)
|
||||
p.add_argument(
|
||||
"--escape-all",
|
||||
action="store_true",
|
||||
dest="escape_snob",
|
||||
default=False,
|
||||
help=(
|
||||
"Escape all special characters. Output is less readable, but avoids "
|
||||
"corner case formatting issues."
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--bypass-tables",
|
||||
action="store_true",
|
||||
dest="bypass_tables",
|
||||
default=config.BYPASS_TABLES,
|
||||
help="Format tables in HTML rather than Markdown syntax.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--ignore-tables",
|
||||
action="store_true",
|
||||
dest="ignore_tables",
|
||||
default=config.IGNORE_TABLES,
|
||||
help="Ignore table-related tags (table, th, td, tr) " "while keeping rows.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--single-line-break",
|
||||
action="store_true",
|
||||
dest="single_line_break",
|
||||
default=config.SINGLE_LINE_BREAK,
|
||||
help=(
|
||||
"Use a single line break after a block element rather than two line "
|
||||
"breaks. NOTE: Requires --body-width=0"
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--unicode-snob",
|
||||
action="store_true",
|
||||
dest="unicode_snob",
|
||||
default=config.UNICODE_SNOB,
|
||||
help="Use unicode throughout document",
|
||||
)
|
||||
p.add_argument(
|
||||
"--no-automatic-links",
|
||||
action="store_false",
|
||||
dest="use_automatic_links",
|
||||
default=config.USE_AUTOMATIC_LINKS,
|
||||
help="Do not use automatic links wherever applicable",
|
||||
)
|
||||
p.add_argument(
|
||||
"--no-skip-internal-links",
|
||||
action="store_false",
|
||||
dest="skip_internal_links",
|
||||
default=config.SKIP_INTERNAL_LINKS,
|
||||
help="Do not skip internal links",
|
||||
)
|
||||
p.add_argument(
|
||||
"--links-after-para",
|
||||
action="store_true",
|
||||
dest="links_each_paragraph",
|
||||
default=config.LINKS_EACH_PARAGRAPH,
|
||||
help="Put links after each paragraph instead of document",
|
||||
)
|
||||
p.add_argument(
|
||||
"--mark-code",
|
||||
action="store_true",
|
||||
dest="mark_code",
|
||||
default=config.MARK_CODE,
|
||||
help="Mark program code blocks with [code]...[/code]",
|
||||
)
|
||||
p.add_argument(
|
||||
"--decode-errors",
|
||||
dest="decode_errors",
|
||||
default=config.DECODE_ERRORS,
|
||||
help=(
|
||||
"What to do in case of decode errors.'ignore', 'strict' and 'replace' are "
|
||||
"acceptable values"
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--open-quote",
|
||||
dest="open_quote",
|
||||
default=config.OPEN_QUOTE,
|
||||
help="The character used to open quotes",
|
||||
)
|
||||
p.add_argument(
|
||||
"--close-quote",
|
||||
dest="close_quote",
|
||||
default=config.CLOSE_QUOTE,
|
||||
help="The character used to close quotes",
|
||||
)
|
||||
p.add_argument(
|
||||
"--version", action="version", version=".".join(map(str, __version__))
|
||||
)
|
||||
p.add_argument("filename", nargs="?")
|
||||
p.add_argument("encoding", nargs="?", default="utf-8")
|
||||
args = p.parse_args()
|
||||
|
||||
if args.filename and args.filename != "-":
|
||||
with open(args.filename, "rb") as fp:
|
||||
data = fp.read()
|
||||
else:
|
||||
data = sys.stdin.buffer.read()
|
||||
|
||||
try:
|
||||
html = data.decode(args.encoding, args.decode_errors)
|
||||
except UnicodeDecodeError as err:
|
||||
warning = bcolors.WARNING + "Warning:" + bcolors.ENDC
|
||||
warning += " Use the " + bcolors.OKGREEN
|
||||
warning += "--decode-errors=ignore" + bcolors.ENDC + " flag."
|
||||
print(warning)
|
||||
raise err
|
||||
|
||||
h = HTML2Text(baseurl=baseurl)
|
||||
# handle options
|
||||
if args.ul_style_dash:
|
||||
h.ul_item_mark = "-"
|
||||
if args.em_style_asterisk:
|
||||
h.emphasis_mark = "*"
|
||||
h.strong_mark = "__"
|
||||
|
||||
h.body_width = args.body_width
|
||||
h.google_list_indent = args.list_indent
|
||||
h.ignore_emphasis = args.ignore_emphasis
|
||||
h.ignore_links = args.ignore_links
|
||||
h.protect_links = args.protect_links
|
||||
h.ignore_images = args.ignore_images
|
||||
h.images_as_html = args.images_as_html
|
||||
h.images_to_alt = args.images_to_alt
|
||||
h.images_with_size = args.images_with_size
|
||||
h.google_doc = args.google_doc
|
||||
h.hide_strikethrough = args.hide_strikethrough
|
||||
h.escape_snob = args.escape_snob
|
||||
h.bypass_tables = args.bypass_tables
|
||||
h.ignore_tables = args.ignore_tables
|
||||
h.single_line_break = args.single_line_break
|
||||
h.inline_links = args.inline_links
|
||||
h.unicode_snob = args.unicode_snob
|
||||
h.use_automatic_links = args.use_automatic_links
|
||||
h.skip_internal_links = args.skip_internal_links
|
||||
h.links_each_paragraph = args.links_each_paragraph
|
||||
h.mark_code = args.mark_code
|
||||
h.wrap_links = args.wrap_links
|
||||
h.wrap_list_items = args.wrap_list_items
|
||||
h.pad_tables = args.pad_tables
|
||||
h.default_image_alt = args.default_image_alt
|
||||
h.open_quote = args.open_quote
|
||||
h.close_quote = args.close_quote
|
||||
|
||||
sys.stdout.write(h.handle(html))
|
||||
158
core/html2text/config.py
Normal file
158
core/html2text/config.py
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
import re
|
||||
|
||||
# Use Unicode characters instead of their ascii pseudo-replacements
|
||||
UNICODE_SNOB = False
|
||||
|
||||
# Marker to use for marking tables for padding post processing
|
||||
TABLE_MARKER_FOR_PAD = "special_marker_for_table_padding"
|
||||
# Escape all special characters. Output is less readable, but avoids
|
||||
# corner case formatting issues.
|
||||
ESCAPE_SNOB = False
|
||||
|
||||
# Put the links after each paragraph instead of at the end.
|
||||
LINKS_EACH_PARAGRAPH = False
|
||||
|
||||
# Wrap long lines at position. 0 for no wrapping.
|
||||
BODY_WIDTH = 78
|
||||
|
||||
# Don't show internal links (href="#local-anchor") -- corresponding link
|
||||
# targets won't be visible in the plain text file anyway.
|
||||
SKIP_INTERNAL_LINKS = True
|
||||
|
||||
# Use inline, rather than reference, formatting for images and links
|
||||
INLINE_LINKS = True
|
||||
|
||||
# Protect links from line breaks surrounding them with angle brackets (in
|
||||
# addition to their square brackets)
|
||||
PROTECT_LINKS = False
|
||||
# WRAP_LINKS = True
|
||||
WRAP_LINKS = True
|
||||
|
||||
# Wrap list items.
|
||||
WRAP_LIST_ITEMS = False
|
||||
|
||||
# Number of pixels Google indents nested lists
|
||||
GOOGLE_LIST_INDENT = 36
|
||||
|
||||
# Values Google and others may use to indicate bold text
|
||||
BOLD_TEXT_STYLE_VALUES = ("bold", "700", "800", "900")
|
||||
|
||||
IGNORE_ANCHORS = False
|
||||
IGNORE_IMAGES = False
|
||||
IMAGES_AS_HTML = False
|
||||
IMAGES_TO_ALT = False
|
||||
IMAGES_WITH_SIZE = False
|
||||
IGNORE_EMPHASIS = False
|
||||
MARK_CODE = False
|
||||
DECODE_ERRORS = "strict"
|
||||
DEFAULT_IMAGE_ALT = ""
|
||||
PAD_TABLES = False
|
||||
|
||||
# Convert links with same href and text to <href> format
|
||||
# if they are absolute links
|
||||
USE_AUTOMATIC_LINKS = True
|
||||
|
||||
# For checking space-only lines on line 771
|
||||
RE_SPACE = re.compile(r"\s\+")
|
||||
|
||||
RE_ORDERED_LIST_MATCHER = re.compile(r"\d+\.\s")
|
||||
RE_UNORDERED_LIST_MATCHER = re.compile(r"[-\*\+]\s")
|
||||
RE_MD_CHARS_MATCHER = re.compile(r"([\\\[\]\(\)])")
|
||||
RE_MD_CHARS_MATCHER_ALL = re.compile(r"([`\*_{}\[\]\(\)#!])")
|
||||
|
||||
# to find links in the text
|
||||
RE_LINK = re.compile(r"(\[.*?\] ?\(.*?\))|(\[.*?\]:.*?)")
|
||||
|
||||
RE_MD_DOT_MATCHER = re.compile(
|
||||
r"""
|
||||
^ # start of line
|
||||
(\s*\d+) # optional whitespace and a number
|
||||
(\.) # dot
|
||||
(?=\s) # lookahead assert whitespace
|
||||
""",
|
||||
re.MULTILINE | re.VERBOSE,
|
||||
)
|
||||
RE_MD_PLUS_MATCHER = re.compile(
|
||||
r"""
|
||||
^
|
||||
(\s*)
|
||||
(\+)
|
||||
(?=\s)
|
||||
""",
|
||||
flags=re.MULTILINE | re.VERBOSE,
|
||||
)
|
||||
RE_MD_DASH_MATCHER = re.compile(
|
||||
r"""
|
||||
^
|
||||
(\s*)
|
||||
(-)
|
||||
(?=\s|\-) # followed by whitespace (bullet list, or spaced out hr)
|
||||
# or another dash (header or hr)
|
||||
""",
|
||||
flags=re.MULTILINE | re.VERBOSE,
|
||||
)
|
||||
RE_SLASH_CHARS = r"\`*_{}[]()#+-.!"
|
||||
RE_MD_BACKSLASH_MATCHER = re.compile(
|
||||
r"""
|
||||
(\\) # match one slash
|
||||
(?=[%s]) # followed by a char that requires escaping
|
||||
"""
|
||||
% re.escape(RE_SLASH_CHARS),
|
||||
flags=re.VERBOSE,
|
||||
)
|
||||
|
||||
UNIFIABLE = {
|
||||
"rsquo": "'",
|
||||
"lsquo": "'",
|
||||
"rdquo": '"',
|
||||
"ldquo": '"',
|
||||
"copy": "(C)",
|
||||
"mdash": "--",
|
||||
"nbsp": " ",
|
||||
"rarr": "->",
|
||||
"larr": "<-",
|
||||
"middot": "*",
|
||||
"ndash": "-",
|
||||
"oelig": "oe",
|
||||
"aelig": "ae",
|
||||
"agrave": "a",
|
||||
"aacute": "a",
|
||||
"acirc": "a",
|
||||
"atilde": "a",
|
||||
"auml": "a",
|
||||
"aring": "a",
|
||||
"egrave": "e",
|
||||
"eacute": "e",
|
||||
"ecirc": "e",
|
||||
"euml": "e",
|
||||
"igrave": "i",
|
||||
"iacute": "i",
|
||||
"icirc": "i",
|
||||
"iuml": "i",
|
||||
"ograve": "o",
|
||||
"oacute": "o",
|
||||
"ocirc": "o",
|
||||
"otilde": "o",
|
||||
"ouml": "o",
|
||||
"ugrave": "u",
|
||||
"uacute": "u",
|
||||
"ucirc": "u",
|
||||
"uuml": "u",
|
||||
"lrm": "",
|
||||
"rlm": "",
|
||||
}
|
||||
|
||||
# Format tables in HTML rather than Markdown syntax
|
||||
BYPASS_TABLES = False
|
||||
# Ignore table-related tags (table, th, td, tr) while keeping rows
|
||||
IGNORE_TABLES = False
|
||||
|
||||
|
||||
# Use a single line break after a block element rather than two line breaks.
|
||||
# NOTE: Requires body width setting to be 0.
|
||||
SINGLE_LINE_BREAK = False
|
||||
|
||||
|
||||
# Use double quotation marks when converting the <q> tag.
|
||||
OPEN_QUOTE = '"'
|
||||
CLOSE_QUOTE = '"'
|
||||
18
core/html2text/elements.py
Normal file
18
core/html2text/elements.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
from typing import Dict, Optional
|
||||
|
||||
|
||||
class AnchorElement:
|
||||
__slots__ = ["attrs", "count", "outcount"]
|
||||
|
||||
def __init__(self, attrs: Dict[str, Optional[str]], count: int, outcount: int):
|
||||
self.attrs = attrs
|
||||
self.count = count
|
||||
self.outcount = outcount
|
||||
|
||||
|
||||
class ListElement:
|
||||
__slots__ = ["name", "num"]
|
||||
|
||||
def __init__(self, name: str, num: int):
|
||||
self.name = name
|
||||
self.num = num
|
||||
0
core/html2text/py.typed
Normal file
0
core/html2text/py.typed
Normal file
3
core/html2text/typing.py
Normal file
3
core/html2text/typing.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
class OutCallback:
|
||||
def __call__(self, s: str) -> None:
|
||||
...
|
||||
283
core/html2text/utils.py
Normal file
283
core/html2text/utils.py
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
import html.entities
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from . import config
|
||||
|
||||
unifiable_n = {
|
||||
html.entities.name2codepoint[k]: v
|
||||
for k, v in config.UNIFIABLE.items()
|
||||
if k != "nbsp"
|
||||
}
|
||||
|
||||
|
||||
def hn(tag: str) -> int:
|
||||
if tag[0] == "h" and len(tag) == 2:
|
||||
n = tag[1]
|
||||
if "0" < n <= "9":
|
||||
return int(n)
|
||||
return 0
|
||||
|
||||
|
||||
def dumb_property_dict(style: str) -> Dict[str, str]:
|
||||
"""
|
||||
:returns: A hash of css attributes
|
||||
"""
|
||||
return {
|
||||
x.strip().lower(): y.strip().lower()
|
||||
for x, y in [z.split(":", 1) for z in style.split(";") if ":" in z]
|
||||
}
|
||||
|
||||
|
||||
def dumb_css_parser(data: str) -> Dict[str, Dict[str, str]]:
|
||||
"""
|
||||
:type data: str
|
||||
|
||||
:returns: A hash of css selectors, each of which contains a hash of
|
||||
css attributes.
|
||||
:rtype: dict
|
||||
"""
|
||||
# remove @import sentences
|
||||
data += ";"
|
||||
importIndex = data.find("@import")
|
||||
while importIndex != -1:
|
||||
data = data[0:importIndex] + data[data.find(";", importIndex) + 1 :]
|
||||
importIndex = data.find("@import")
|
||||
|
||||
# parse the css. reverted from dictionary comprehension in order to
|
||||
# support older pythons
|
||||
pairs = [x.split("{") for x in data.split("}") if "{" in x.strip()]
|
||||
try:
|
||||
elements = {a.strip(): dumb_property_dict(b) for a, b in pairs}
|
||||
except ValueError:
|
||||
elements = {} # not that important
|
||||
|
||||
return elements
|
||||
|
||||
|
||||
def element_style(
|
||||
attrs: Dict[str, Optional[str]],
|
||||
style_def: Dict[str, Dict[str, str]],
|
||||
parent_style: Dict[str, str],
|
||||
) -> Dict[str, str]:
|
||||
"""
|
||||
:type attrs: dict
|
||||
:type style_def: dict
|
||||
:type style_def: dict
|
||||
|
||||
:returns: A hash of the 'final' style attributes of the element
|
||||
:rtype: dict
|
||||
"""
|
||||
style = parent_style.copy()
|
||||
if "class" in attrs:
|
||||
assert attrs["class"] is not None
|
||||
for css_class in attrs["class"].split():
|
||||
css_style = style_def.get("." + css_class, {})
|
||||
style.update(css_style)
|
||||
if "style" in attrs:
|
||||
assert attrs["style"] is not None
|
||||
immediate_style = dumb_property_dict(attrs["style"])
|
||||
style.update(immediate_style)
|
||||
|
||||
return style
|
||||
|
||||
|
||||
def google_list_style(style: Dict[str, str]) -> str:
|
||||
"""
|
||||
Finds out whether this is an ordered or unordered list
|
||||
|
||||
:type style: dict
|
||||
|
||||
:rtype: str
|
||||
"""
|
||||
if "list-style-type" in style:
|
||||
list_style = style["list-style-type"]
|
||||
if list_style in ["disc", "circle", "square", "none"]:
|
||||
return "ul"
|
||||
|
||||
return "ol"
|
||||
|
||||
|
||||
def google_has_height(style: Dict[str, str]) -> bool:
|
||||
"""
|
||||
Check if the style of the element has the 'height' attribute
|
||||
explicitly defined
|
||||
|
||||
:type style: dict
|
||||
|
||||
:rtype: bool
|
||||
"""
|
||||
return "height" in style
|
||||
|
||||
|
||||
def google_text_emphasis(style: Dict[str, str]) -> List[str]:
|
||||
"""
|
||||
:type style: dict
|
||||
|
||||
:returns: A list of all emphasis modifiers of the element
|
||||
:rtype: list
|
||||
"""
|
||||
emphasis = []
|
||||
if "text-decoration" in style:
|
||||
emphasis.append(style["text-decoration"])
|
||||
if "font-style" in style:
|
||||
emphasis.append(style["font-style"])
|
||||
if "font-weight" in style:
|
||||
emphasis.append(style["font-weight"])
|
||||
|
||||
return emphasis
|
||||
|
||||
|
||||
def google_fixed_width_font(style: Dict[str, str]) -> bool:
|
||||
"""
|
||||
Check if the css of the current element defines a fixed width font
|
||||
|
||||
:type style: dict
|
||||
|
||||
:rtype: bool
|
||||
"""
|
||||
font_family = ""
|
||||
if "font-family" in style:
|
||||
font_family = style["font-family"]
|
||||
return "courier new" == font_family or "consolas" == font_family
|
||||
|
||||
|
||||
def list_numbering_start(attrs: Dict[str, Optional[str]]) -> int:
|
||||
"""
|
||||
Extract numbering from list element attributes
|
||||
|
||||
:type attrs: dict
|
||||
|
||||
:rtype: int or None
|
||||
"""
|
||||
if "start" in attrs:
|
||||
assert attrs["start"] is not None
|
||||
try:
|
||||
return int(attrs["start"]) - 1
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def skipwrap(para: str, wrap_links: bool, wrap_list_items: bool) -> bool:
|
||||
# If it appears to contain a link
|
||||
# don't wrap
|
||||
if not wrap_links and config.RE_LINK.search(para):
|
||||
return True
|
||||
# If the text begins with four spaces or one tab, it's a code block;
|
||||
# don't wrap
|
||||
if para[0:4] == " " or para[0] == "\t":
|
||||
return True
|
||||
|
||||
# If the text begins with only two "--", possibly preceded by
|
||||
# whitespace, that's an emdash; so wrap.
|
||||
stripped = para.lstrip()
|
||||
if stripped[0:2] == "--" and len(stripped) > 2 and stripped[2] != "-":
|
||||
return False
|
||||
|
||||
# I'm not sure what this is for; I thought it was to detect lists,
|
||||
# but there's a <br>-inside-<span> case in one of the tests that
|
||||
# also depends upon it.
|
||||
if stripped[0:1] in ("-", "*") and not stripped[0:2] == "**":
|
||||
return not wrap_list_items
|
||||
|
||||
# If the text begins with a single -, *, or +, followed by a space,
|
||||
# or an integer, followed by a ., followed by a space (in either
|
||||
# case optionally proceeded by whitespace), it's a list; don't wrap.
|
||||
return bool(
|
||||
config.RE_ORDERED_LIST_MATCHER.match(stripped)
|
||||
or config.RE_UNORDERED_LIST_MATCHER.match(stripped)
|
||||
)
|
||||
|
||||
|
||||
def escape_md(text: str) -> str:
|
||||
"""
|
||||
Escapes markdown-sensitive characters within other markdown
|
||||
constructs.
|
||||
"""
|
||||
return config.RE_MD_CHARS_MATCHER.sub(r"\\\1", text)
|
||||
|
||||
|
||||
def escape_md_section(text: str, snob: bool = False) -> str:
|
||||
"""
|
||||
Escapes markdown-sensitive characters across whole document sections.
|
||||
"""
|
||||
text = config.RE_MD_BACKSLASH_MATCHER.sub(r"\\\1", text)
|
||||
|
||||
if snob:
|
||||
text = config.RE_MD_CHARS_MATCHER_ALL.sub(r"\\\1", text)
|
||||
|
||||
text = config.RE_MD_DOT_MATCHER.sub(r"\1\\\2", text)
|
||||
text = config.RE_MD_PLUS_MATCHER.sub(r"\1\\\2", text)
|
||||
text = config.RE_MD_DASH_MATCHER.sub(r"\1\\\2", text)
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def reformat_table(lines: List[str], right_margin: int) -> List[str]:
|
||||
"""
|
||||
Given the lines of a table
|
||||
padds the cells and returns the new lines
|
||||
"""
|
||||
# find the maximum width of the columns
|
||||
max_width = [len(x.rstrip()) + right_margin for x in lines[0].split("|")]
|
||||
max_cols = len(max_width)
|
||||
for line in lines:
|
||||
cols = [x.rstrip() for x in line.split("|")]
|
||||
num_cols = len(cols)
|
||||
|
||||
# don't drop any data if colspan attributes result in unequal lengths
|
||||
if num_cols < max_cols:
|
||||
cols += [""] * (max_cols - num_cols)
|
||||
elif max_cols < num_cols:
|
||||
max_width += [len(x) + right_margin for x in cols[-(num_cols - max_cols) :]]
|
||||
max_cols = num_cols
|
||||
|
||||
max_width = [
|
||||
max(len(x) + right_margin, old_len) for x, old_len in zip(cols, max_width)
|
||||
]
|
||||
|
||||
# reformat
|
||||
new_lines = []
|
||||
for line in lines:
|
||||
cols = [x.rstrip() for x in line.split("|")]
|
||||
if set(line.strip()) == set("-|"):
|
||||
filler = "-"
|
||||
new_cols = [
|
||||
x.rstrip() + (filler * (M - len(x.rstrip())))
|
||||
for x, M in zip(cols, max_width)
|
||||
]
|
||||
else:
|
||||
filler = " "
|
||||
new_cols = [
|
||||
x.rstrip() + (filler * (M - len(x.rstrip())))
|
||||
for x, M in zip(cols, max_width)
|
||||
]
|
||||
new_lines.append("|".join(new_cols))
|
||||
return new_lines
|
||||
|
||||
|
||||
def pad_tables_in_text(text: str, right_margin: int = 1) -> str:
|
||||
"""
|
||||
Provide padding for tables in the text
|
||||
"""
|
||||
lines = text.split("\n")
|
||||
table_buffer = [] # type: List[str]
|
||||
table_started = False
|
||||
new_lines = []
|
||||
for line in lines:
|
||||
# Toggle table started
|
||||
if config.TABLE_MARKER_FOR_PAD in line:
|
||||
table_started = not table_started
|
||||
if not table_started:
|
||||
table = reformat_table(table_buffer, right_margin)
|
||||
new_lines.extend(table)
|
||||
table_buffer = []
|
||||
new_lines.append("")
|
||||
continue
|
||||
# Process lines
|
||||
if table_started:
|
||||
table_buffer.append(line)
|
||||
else:
|
||||
new_lines.append(line)
|
||||
return "\n".join(new_lines)
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import feedparser
|
||||
import html2text
|
||||
from core.html2text import html2text
|
||||
from core.utils.bot import get_url
|
||||
|
||||
|
||||
|
|
@ -11,5 +11,5 @@ async def get_rss():
|
|||
status_code=200, fmt='text')
|
||||
feed = feedparser.parse(url)['entries'][-1]
|
||||
title = feed['title']
|
||||
summary = html2text.html2text(feed['summary'], baseurl='https://lakeus.xyz')
|
||||
summary = html2text(feed['summary'], baseurl='https://lakeus.xyz')
|
||||
return title + '\n' + summary
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import traceback
|
|||
import urllib.parse
|
||||
from typing import Union, Dict, List
|
||||
|
||||
import html2text
|
||||
import core.html2text as html2text
|
||||
import ujson as json
|
||||
|
||||
from core.dirty_check import check
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ eventlet
|
|||
psutil
|
||||
gql[all]
|
||||
graiax-silkcoder
|
||||
html2text
|
||||
graia-scheduler
|
||||
filetype
|
||||
sqlalchemy
|
||||
|
|
|
|||
Reference in a new issue