added optionals and translation logic to containers
This commit is contained in:
parent
67bb38b77b
commit
ab2bc924cc
|
|
@ -11,6 +11,7 @@
|
||||||
# container = BaseContainer()
|
# container = BaseContainer()
|
||||||
# container[number] -> The number element in the content
|
# container[number] -> The number element in the content
|
||||||
# "class_insert" >> container[number] -> Get an element from one of the mapped methods
|
# "class_insert" >> container[number] -> Get an element from one of the mapped methods
|
||||||
|
import rich
|
||||||
|
|
||||||
from bootstraparse.modules import syntax, error_mngr, export
|
from bootstraparse.modules import syntax, error_mngr, export
|
||||||
from bootstraparse.modules.error_mngr import MismatchedContainerError, log_exception, log_message # noqa
|
from bootstraparse.modules.error_mngr import MismatchedContainerError, log_exception, log_message # noqa
|
||||||
|
|
@ -20,12 +21,35 @@ class BaseContainer:
|
||||||
"""
|
"""
|
||||||
Creates container holding all the elements from the start of a parsed element to its end.
|
Creates container holding all the elements from the start of a parsed element to its end.
|
||||||
"""
|
"""
|
||||||
|
type = None
|
||||||
|
subtype = None
|
||||||
|
|
||||||
def __init__(self, content=None):
|
def __init__(self, content=None):
|
||||||
if content is None:
|
if content is None:
|
||||||
content = []
|
content = []
|
||||||
self.content = content
|
self.content = content
|
||||||
self.optionals = []
|
|
||||||
self.map = {}
|
self.map = {}
|
||||||
|
self.optionals = None
|
||||||
|
|
||||||
|
def get_content(self, exm):
|
||||||
|
output = ""
|
||||||
|
for element in self.content:
|
||||||
|
if isinstance(element, BaseContainer):
|
||||||
|
output += element.export(exm)
|
||||||
|
return output
|
||||||
|
|
||||||
|
def get_optionals(self):
|
||||||
|
return self.optionals
|
||||||
|
|
||||||
|
def get_others(self): # noqa
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def export(self, exm):
|
||||||
|
start, end = exm(export.ExportRequest(self.type, self.subtype, self.get_optionals(), self.get_others()))
|
||||||
|
|
||||||
|
output = start + self.get_content(exm) + end
|
||||||
|
|
||||||
|
return output
|
||||||
|
|
||||||
def add(self, other):
|
def add(self, other):
|
||||||
self.content.append(other)
|
self.content.append(other)
|
||||||
|
|
@ -48,9 +72,6 @@ class BaseContainer:
|
||||||
for k, v in self.map.items():
|
for k, v in self.map.items():
|
||||||
print(f'{k} = {v}')
|
print(f'{k} = {v}')
|
||||||
|
|
||||||
def export(self, exm):
|
|
||||||
return ""
|
|
||||||
|
|
||||||
def __len__(self):
|
def __len__(self):
|
||||||
return len(self.content)
|
return len(self.content)
|
||||||
|
|
||||||
|
|
@ -81,11 +102,11 @@ class BaseContainer:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def __invert__(self):
|
def __invert__(self):
|
||||||
return self.optionals
|
return self.get_optionals()
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return '{} ({})'.format(", ".join(map(str, self.content)),
|
return '{} ({})'.format(", ".join(map(str, self.content)),
|
||||||
", ".join(map(str, self.optionals)) if self.optionals != [] else "")
|
", ".join(map(str, self.get_optionals())) if self.get_optionals() != [] else "")
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
representation = {}
|
representation = {}
|
||||||
|
|
@ -110,7 +131,7 @@ class BaseContainer:
|
||||||
print(ident + "> " + str(e))
|
print(ident + "> " + str(e))
|
||||||
print("")
|
print("")
|
||||||
|
|
||||||
def to_container(self):
|
def to_container(self): # idea: additional parameters for encapsulation
|
||||||
return self
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -120,13 +141,21 @@ class BaseContainerWithOptionals(BaseContainer):
|
||||||
self.map['html_insert'] = self.fetch_html_insert
|
self.map['html_insert'] = self.fetch_html_insert
|
||||||
self.map['class_insert'] = self.fetch_class_insert
|
self.map['class_insert'] = self.fetch_class_insert
|
||||||
|
|
||||||
|
def get_optionals(self):
|
||||||
|
rich.inspect(self.content)
|
||||||
|
rich.inspect(self.content[0])
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_others(self):
|
||||||
|
return None
|
||||||
|
|
||||||
def fetch_html_insert(self):
|
def fetch_html_insert(self):
|
||||||
"""
|
"""
|
||||||
Returns a list of all html_inserts, and an empty list if none exist.
|
Returns a list of all html_inserts, and an empty list if none exist.
|
||||||
:rtype: list
|
:rtype: list
|
||||||
"""
|
"""
|
||||||
rlist = []
|
rlist = []
|
||||||
for o in self.optionals:
|
for o in self.get_optionals():
|
||||||
if o.label == 'optional:insert':
|
if o.label == 'optional:insert':
|
||||||
rlist += [o]
|
rlist += [o]
|
||||||
return rlist
|
return rlist
|
||||||
|
|
@ -137,7 +166,7 @@ class BaseContainerWithOptionals(BaseContainer):
|
||||||
:rtype: list
|
:rtype: list
|
||||||
"""
|
"""
|
||||||
rlist = []
|
rlist = []
|
||||||
for o in self.optionals:
|
for o in self.get_optionals():
|
||||||
if o.label == 'optional:class':
|
if o.label == 'optional:class':
|
||||||
rlist += [o]
|
rlist += [o]
|
||||||
return rlist
|
return rlist
|
||||||
|
|
@ -155,52 +184,47 @@ class TextContainer(BaseContainer):
|
||||||
return output
|
return output
|
||||||
|
|
||||||
|
|
||||||
class EtEmContainer(BaseContainer): # TODO: move methods to BaseContainer, deal with attributes individually
|
class EtEmContainer(BaseContainer):
|
||||||
type = "inline_elements"
|
type = "inline_elements"
|
||||||
subtype = "em"
|
subtype = "em"
|
||||||
optionals = ''
|
|
||||||
others = ''
|
|
||||||
|
|
||||||
def get_content(self, exm):
|
|
||||||
output = ""
|
|
||||||
for element in self.content:
|
|
||||||
if isinstance(element, BaseContainer):
|
|
||||||
output += element.export(exm)
|
|
||||||
return output
|
|
||||||
|
|
||||||
def export(self, exm):
|
|
||||||
start, end = exm(export.ExportRequest(self.type, self.subtype, self.optionals, self.others))
|
|
||||||
|
|
||||||
output = start + self.get_content(exm) + end
|
|
||||||
|
|
||||||
return output
|
|
||||||
|
|
||||||
|
|
||||||
class EtStrongContainer(BaseContainer):
|
class EtStrongContainer(BaseContainer):
|
||||||
pass
|
type = "inline_elements"
|
||||||
|
subtype = "strong"
|
||||||
|
|
||||||
|
|
||||||
class EtUnderlineContainer(BaseContainer):
|
class EtUnderlineContainer(BaseContainer):
|
||||||
pass
|
type = "inline_elements"
|
||||||
|
subtype = "underline"
|
||||||
|
|
||||||
|
|
||||||
class EtStrikethroughContainer(BaseContainer):
|
class EtStrikethroughContainer(BaseContainer):
|
||||||
pass
|
type = "inline_elements"
|
||||||
|
subtype = "strikethrough"
|
||||||
|
|
||||||
|
|
||||||
class EtCustomSpanContainer(BaseContainer):
|
class EtCustomSpanContainer(BaseContainer):
|
||||||
pass
|
type = "inline_elements"
|
||||||
|
subtype = "custom_0" # future: add logic
|
||||||
|
|
||||||
|
|
||||||
class EtUlistContainer(BaseContainer):
|
class EtUlistContainer(BaseContainer):
|
||||||
pass
|
type = "oneline_elements"
|
||||||
|
subtype = "ulist"
|
||||||
|
children = "list_line"
|
||||||
|
|
||||||
|
|
||||||
class EtOlistContainer(BaseContainer):
|
class EtOlistContainer(BaseContainer):
|
||||||
pass
|
type = "oneline_elements"
|
||||||
|
subtype = "olist"
|
||||||
|
children = "list_line"
|
||||||
|
|
||||||
|
|
||||||
class HyperLinkContainer(BaseContainer):
|
class HyperLinkContainer(BaseContainer):
|
||||||
|
type = "inline_elements"
|
||||||
|
subtype = "link"
|
||||||
|
|
||||||
def __init__(self, content=None):
|
def __init__(self, content=None):
|
||||||
super().__init__(content)
|
super().__init__(content)
|
||||||
self.map['url'] = ""
|
self.map['url'] = ""
|
||||||
|
|
@ -212,10 +236,14 @@ class HyperLinkContainer(BaseContainer):
|
||||||
|
|
||||||
|
|
||||||
class SeContainer(BaseContainer):
|
class SeContainer(BaseContainer):
|
||||||
pass
|
type = "structural_elements"
|
||||||
|
subtype = "div" # TODO: add logic
|
||||||
|
|
||||||
|
|
||||||
class HeaderContainer(BaseContainer):
|
class HeaderContainer(BaseContainer):
|
||||||
|
type = "structural_elements"
|
||||||
|
subtype = "header"
|
||||||
|
|
||||||
def __init__(self, content=None):
|
def __init__(self, content=None):
|
||||||
super().__init__(content)
|
super().__init__(content)
|
||||||
self.map['header_level'] = ''
|
self.map['header_level'] = ''
|
||||||
|
|
@ -223,6 +251,9 @@ class HeaderContainer(BaseContainer):
|
||||||
|
|
||||||
|
|
||||||
class DisplayContainer(BaseContainer):
|
class DisplayContainer(BaseContainer):
|
||||||
|
type = "structural_elements"
|
||||||
|
subtype = "display"
|
||||||
|
|
||||||
def __init__(self, content=None):
|
def __init__(self, content=None):
|
||||||
super().__init__(content)
|
super().__init__(content)
|
||||||
self.map['display_level'] = ""
|
self.map['display_level'] = ""
|
||||||
|
|
@ -234,6 +265,9 @@ class TableSeparatorContainer(BaseContainer):
|
||||||
|
|
||||||
|
|
||||||
# class TableHeadContainer(BaseContainer):
|
# class TableHeadContainer(BaseContainer):
|
||||||
|
# type = "table"
|
||||||
|
# subtype = "t_head"
|
||||||
|
#
|
||||||
# def __init__(self, content=None):
|
# def __init__(self, content=None):
|
||||||
# super().__init__(content)
|
# super().__init__(content)
|
||||||
# self.map['colspan'] = ""
|
# self.map['colspan'] = ""
|
||||||
|
|
@ -241,6 +275,9 @@ class TableSeparatorContainer(BaseContainer):
|
||||||
|
|
||||||
|
|
||||||
class TableRowContainer(BaseContainer):
|
class TableRowContainer(BaseContainer):
|
||||||
|
type = "table"
|
||||||
|
subtype = "t_row"
|
||||||
|
|
||||||
def __init__(self, content=None):
|
def __init__(self, content=None):
|
||||||
super().__init__(content)
|
super().__init__(content)
|
||||||
self.map['colspan'] = ""
|
self.map['colspan'] = ""
|
||||||
|
|
@ -248,6 +285,9 @@ class TableRowContainer(BaseContainer):
|
||||||
|
|
||||||
|
|
||||||
class TableCellContainer(BaseContainer):
|
class TableCellContainer(BaseContainer):
|
||||||
|
type = "table"
|
||||||
|
subtype = "t_cell"
|
||||||
|
|
||||||
def __init__(self, content=None):
|
def __init__(self, content=None):
|
||||||
super().__init__(content)
|
super().__init__(content)
|
||||||
self.map['colspan'] = ""
|
self.map['colspan'] = ""
|
||||||
|
|
@ -302,7 +342,7 @@ class ContextManager:
|
||||||
self.pile = []
|
self.pile = []
|
||||||
self.matched_elements = {}
|
self.matched_elements = {}
|
||||||
self.dict_lookahead = {
|
self.dict_lookahead = {
|
||||||
"list:ulist": ["list:ulist"],
|
# "list:ulist": ["list:ulist"],
|
||||||
"list:olist": ["list:olist"],
|
"list:olist": ["list:olist"],
|
||||||
"table:row": ["table:separator", "table:row"] # TODO: Implement tables
|
"table:row": ["table:separator", "table:row"] # TODO: Implement tables
|
||||||
}
|
}
|
||||||
|
|
@ -348,8 +388,8 @@ class ContextManager:
|
||||||
raise error
|
raise error
|
||||||
for i in range(start, end):
|
for i in range(start, end):
|
||||||
if self.pile[i]:
|
if self.pile[i]:
|
||||||
# container.add(self.pile[i].to_container()) # This line transform the self modifiying containers # MONITOR
|
# container.add(self.pile[i].to_container()) # This line transform the self modifiying containers # MONITOR # noqa
|
||||||
container.add(self.pile[i]) # Could also ignore the first and last element, or ignore the self modifying tokens # noqa : F821
|
container.add(self.pile[i]) # TODO: ignore the first and last element, or ignore the self modifying tokens # noqa : F821
|
||||||
self.pile[i] = None
|
self.pile[i] = None
|
||||||
container.add(self.pile[end])
|
container.add(self.pile[end])
|
||||||
self.pile[end] = None
|
self.pile[end] = None
|
||||||
|
|
@ -384,6 +424,10 @@ class ContextManager:
|
||||||
self.encapsulate(index, index)
|
self.encapsulate(index, index)
|
||||||
line_number += 1
|
line_number += 1
|
||||||
|
|
||||||
|
elif isinstance(token, syntax.OptionalToken):
|
||||||
|
self.get_last_container_in_pile(index).optionals = token
|
||||||
|
self.pile[index] = None
|
||||||
|
|
||||||
elif token.label in self.dict_lookahead: # group together multiple one-lines
|
elif token.label in self.dict_lookahead: # group together multiple one-lines
|
||||||
lookahead_return = self.lookahead(token, index)
|
lookahead_return = self.lookahead(token, index)
|
||||||
index += lookahead_return[0]
|
index += lookahead_return[0]
|
||||||
|
|
@ -406,20 +450,28 @@ class ContextManager:
|
||||||
error_mngr.log_exception(e, level="CRITICAL") # FUTURE: Be more specific.
|
error_mngr.log_exception(e, level="CRITICAL") # FUTURE: Be more specific.
|
||||||
index += 1
|
index += 1
|
||||||
|
|
||||||
# TODO : make this cleanup in a separate function
|
return self.finalize_pile()
|
||||||
final_pile = []
|
|
||||||
for p in self.pile: # cleanses the pile from Nones
|
|
||||||
if p is not None:
|
|
||||||
final_pile.append(p)
|
|
||||||
|
|
||||||
self.pile = final_pile
|
|
||||||
return self.pile
|
|
||||||
|
|
||||||
def __iter__(self):
|
def __iter__(self):
|
||||||
for e in self.pile:
|
for e in self.pile:
|
||||||
if e:
|
if e:
|
||||||
yield e
|
yield e
|
||||||
|
|
||||||
|
def finalize_pile(self):
|
||||||
|
"""
|
||||||
|
Function for cleaning up of the pile after full contextualisation.
|
||||||
|
Removes Nones and checks for any errors or illogical containers.
|
||||||
|
"""
|
||||||
|
final_pile = []
|
||||||
|
for p in self.pile:
|
||||||
|
if p is not None:
|
||||||
|
final_pile.append(p)
|
||||||
|
|
||||||
|
self.pile = final_pile
|
||||||
|
|
||||||
|
# TODO: add cleanup of non-matched elements
|
||||||
|
return self.pile
|
||||||
|
|
||||||
def lookahead(self, token, index):
|
def lookahead(self, token, index):
|
||||||
"""
|
"""
|
||||||
Iterates the pile beginning from index and looks for all tokens matching labels with token.
|
Iterates the pile beginning from index and looks for all tokens matching labels with token.
|
||||||
|
|
@ -445,6 +497,27 @@ class ContextManager:
|
||||||
self.encapsulate(index, index + range_to_encapsulate)
|
self.encapsulate(index, index + range_to_encapsulate)
|
||||||
return range_to_encapsulate, line_skipped
|
return range_to_encapsulate, line_skipped
|
||||||
|
|
||||||
|
def get_last_container_in_pile(self, index):
|
||||||
|
"""
|
||||||
|
Returns the last element in the pile if it is a Container, raises an error otherwise.
|
||||||
|
Return
|
||||||
|
------
|
||||||
|
BaseContainer
|
||||||
|
|
||||||
|
Raises
|
||||||
|
------
|
||||||
|
error_mngr.LonelyOptionalError
|
||||||
|
"""
|
||||||
|
i = index-1 # skip last token as it is self
|
||||||
|
while i >= 0:
|
||||||
|
if self.pile[i]:
|
||||||
|
if isinstance(self.pile[i], BaseContainer):
|
||||||
|
return self.pile[i]
|
||||||
|
else:
|
||||||
|
log_exception(LonelyOptionalError(self.pile[index], self.pile[i]), level="CRITICAL")
|
||||||
|
i -= 1
|
||||||
|
log_exception(LonelyOptionalError(self.pile[index], None), level="CRITICAL")
|
||||||
|
|
||||||
def print_all(self):
|
def print_all(self):
|
||||||
for e in self:
|
for e in self:
|
||||||
if isinstance(e, BaseContainer):
|
if isinstance(e, BaseContainer):
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue