Compare commits

...

111 Commits
V1.0.0 ... main

Author SHA1 Message Date
Maximilien Rigaut 7b7d1aaa39
Merge pull request #42 from idle-org/release
Release 1.0.4
2022-06-16 04:04:38 +02:00
Maximilien Rigaut bb5f9aa4b6
Merge pull request #41 from idle-org/develop
Develop
2022-06-16 04:00:27 +02:00
Maximilien Rigaut 4036dbe05b
Merge branch 'release' into develop 2022-06-16 03:57:04 +02:00
Maximilien Rigaut a6e44b5611
Merge pull request #40 from idle-org/releases/v1.0.4
Version 1.0.4
2022-06-16 03:55:47 +02:00
Maximilien Rigaut d68b2295e8
Version 1.0.4 2022-06-16 03:54:10 +02:00
Maximilien Rigaut 6b26a5c700
Merge pull request #39 from idle-org/features/roadmap
Roadmap
2022-06-16 03:40:25 +02:00
Maximilien Rigaut 0da3828cd0
Roadmap 2022-06-16 03:39:57 +02:00
Maximilien Rigaut 4bfd730616
Merge pull request #38 from idle-org/features/context
Added a better context to error
2022-06-16 03:39:34 +02:00
Maximilien Rigaut 86891e9e86
Added a better context to error 2022-06-16 03:38:27 +02:00
Maximilien Rigaut 09c8eef46b
Merge pull request #37 from idle-org/features/aliases
Inline image & aliases (and index corrected accordingly)
2022-06-16 02:26:54 +02:00
Maximilien Rigaut c2a44f40c5
Inline image & aliases (and index corrected accordingly) 2022-06-16 02:26:13 +02:00
Maximilien Rigaut 237a417396
Merge pull request #36 from idle-org/test-new-doc
Test new doc
2022-06-16 02:04:57 +02:00
Maximilien Rigaut b47c88ffde
Quick fix 2022-06-16 01:47:08 +02:00
Maximilien Rigaut 4a1857c80f
Hacks 2022-06-16 01:34:48 +02:00
Maximilien Rigaut 3d8c1bf4b4
Indent 2022-06-16 01:02:51 +02:00
Maximilien Rigaut 22b19d4481
Indent 2022-06-16 00:50:24 +02:00
Maximilien Rigaut 62da1bacc3
Reformated doc 2022-06-16 00:42:27 +02:00
Maximilien Rigaut ee61ac9b00
test doc 2022-06-16 00:31:49 +02:00
Maximilien Rigaut af71c12b21
Merge pull request #35 from idle-org/release
Release
2022-06-15 23:57:43 +02:00
Maximilien Rigaut aad46ec9f9
Merge pull request #34 from idle-org/develop
Develop
2022-06-15 23:57:14 +02:00
Maximilien Rigaut b420c89ab8
Merge pull request #33 from idle-org/features/header-fix
fixed the header / structural elements export syntax
2022-06-15 23:55:04 +02:00
Maximilien Rigaut 001abba7bf
Merge pull request #30 from idle-org/release
Release 1.0.3
2022-06-15 22:28:44 +02:00
Maximilien Rigaut 77475d3414
fixed the header / structural elements export syntax 2022-06-15 22:25:36 +02:00
Maximilien Rigaut 00d5529930
Merge pull request #29 from idle-org/develop
V1.0.3
2022-06-15 22:00:52 +02:00
Maximilien Rigaut e3f41eee43
Merge branch 'release' into develop 2022-06-15 22:00:20 +02:00
Maximilien Rigaut 3f3d0f7b0f
Merge pull request #28 from idle-org/test-release1.0.3
Preparation for V1.0.3
2022-06-15 21:56:43 +02:00
Maximilien Rigaut ca13bda9c5
Preparation for V1.0.3 2022-06-15 21:52:47 +02:00
Maximilien Rigaut 8989e05af7
Merge branch 'release' 2022-06-15 21:40:06 +02:00
Maximilien Rigaut 64a2763053
Merge branch 'develop' into release 2022-06-15 21:39:48 +02:00
Maximilien Rigaut eeb5c1bb93
Merge pull request #27 from idle-org/test-corrections
hotfix2:deploy
2022-06-15 21:37:24 +02:00
Maximilien Rigaut 6e4babfa14
hotfix2:deploy 2022-06-15 21:32:47 +02:00
Maximilien Rigaut f7a9d38298
Merge pull request #24 from idle-org/develop
hotfix:deploy
2022-06-15 21:09:59 +02:00
Maximilien Rigaut 0d337e5c0c
Merge pull request #23 from idle-org/corrections
Hotfix:deploy
2022-06-15 21:05:48 +02:00
Maximilien Rigaut 5c104b155f
Hotfix:deploy 2022-06-15 21:04:17 +02:00
Maximilien Rigaut d7af2e213a
Merge pull request #21 from idle-org/release
Release
2022-06-15 20:24:14 +02:00
Vicky 48f8d67736
Merge branch 'main' into release 2022-06-15 20:04:31 +02:00
Vicky 0ca9032d15
Merge pull request #20 from idle-org/develop
Develop
2022-06-15 20:01:42 +02:00
Vicky 53455e8ec2
Merge pull request #19 from idle-org/releases/1.0.1
Releases/1.0.1
2022-06-15 19:50:05 +02:00
Maximilien Rigaut 3d0d444181
Applied version and fixes 2022-06-15 19:45:06 +02:00
Maximilien Rigaut 1c8ee2d6e8
Merge branch 'develop' into releases/1.0.1 2022-06-15 19:38:05 +02:00
Maximilien Rigaut 2f4d4ab973
Fixed awful print leftover 2022-06-15 19:23:01 +02:00
Maximilien Rigaut 30e0486b7c
hotfix on header / display bad optionnals 2022-06-15 19:23:01 +02:00
Maximilien Rigaut d94d6dbaed
Hotfix on the syntax with ulits 2022-06-15 19:23:01 +02:00
Maximilien Rigaut 4de49410a0
Added a couple of things inside << and >> detection (body, nav, header) 2022-06-15 19:23:00 +02:00
Maximilien Rigaut bd2fdd8519
Purged regex from requirements.txt 2022-06-15 19:23:00 +02:00
Maximilien Rigaut 92aa25d7dd
Removed the regex requirement 2022-06-15 19:23:00 +02:00
Maximilien Rigaut 84d8fad28f
Added copy behaviour to config file 2022-06-15 19:23:00 +02:00
Maximilien Rigaut ac9f00f363
Test copy feature 2022-06-15 19:23:00 +02:00
Maximilien Rigaut d2390563e4
Copy feature 2022-06-15 19:23:00 +02:00
Maximilien Rigaut 36e56afa4c
Fixed weird code duplication 2022-06-15 19:23:00 +02:00
Maximilien Rigaut 611617a8fd
syntax.py : Doc and rtypes 2022-06-15 19:23:00 +02:00
Maximilien Rigaut 95173d5071
sitecreator.py : Doc and rtypes 2022-06-15 19:22:56 +02:00
Maximilien Rigaut 8305f440ab
Test doc 2022-06-15 19:22:28 +02:00
Maximilien Rigaut 0f8c9e0f5e
sitecrawler.py : Doc and rtypes 2022-06-15 19:22:28 +02:00
Maximilien Rigaut c9d02dee1a
Continuous integration of list 2022-06-15 19:22:21 +02:00
Maximilien Rigaut fce4b4b4c0
preparser.py : Doc and rtypes 2022-06-15 19:21:33 +02:00
Maximilien Rigaut e9b318b4c6
Typos 2022-06-15 19:21:33 +02:00
Maximilien Rigaut 5d71bc3710
Added __init__.py 2022-06-15 19:21:33 +02:00
Maximilien Rigaut 69822173d4
pathresolver.py : rtype and function doc (part) 2022-06-15 19:21:33 +02:00
Maximilien Rigaut 0130cbf740
export.py : rtype and function doc (part) 2022-06-15 19:21:33 +02:00
Maximilien Rigaut 7c6504d0aa
error_mngr.py : rtype and function doc (part) 2022-06-15 19:21:26 +02:00
Maximilien Rigaut 92ee19a1ac
environment.py : rtype and function doc (part) 2022-06-15 19:20:38 +02:00
Vicky 79d22b18ad
Update index.bpr
Corrected [] and {}
2022-06-15 19:18:59 +02:00
Maximilien Rigaut e420ff8e5c
context_mngr.py : rtype and function doc (part) 2022-06-15 19:18:59 +02:00
Maximilien Rigaut 3e1c8b8f19
PEP 2022-06-15 19:18:59 +02:00
Maximilien Rigaut ed330e66a4
config: rtype and function doc 2022-06-15 19:18:59 +02:00
vicky 3c166d07e1
minor code refactor 2022-06-15 19:18:59 +02:00
Maximilien Rigaut 06085a1398
Fixed awful print leftover 2022-06-15 18:53:52 +02:00
Maximilien Rigaut 2516e10cc1
hotfix on header / display bad optionnals 2022-06-15 18:46:27 +02:00
Maximilien Rigaut 1ef5907f0f
Hotfix on the syntax with ulits 2022-06-15 18:46:27 +02:00
Maximilien Rigaut f4e16a0f3b
Renamed actions 2022-06-15 18:02:03 +02:00
Maximilien Rigaut 19ddd40134
Another emergency push to release 2022-06-15 17:53:35 +02:00
Maximilien Rigaut 129b540d36
Applied an emergency test update 2022-06-15 17:38:07 +02:00
Maximilien Rigaut d87f3c127c
Added a couple of things inside << and >> detection (body, nav, header) 2022-06-15 13:26:00 +02:00
Vicky 909d5c2d67
Update README.md
added lead token to roadmap
2022-06-15 13:23:52 +02:00
Vicky a2ae3c5d0e
Merge pull request #5 from idle-org/features/reguekes
Features/reguekes
2022-06-13 19:24:23 +02:00
Maximilien Rigaut 7ba4af8d45
Purged regex from requirements.txt 2022-06-13 19:15:15 +02:00
Maximilien Rigaut eb00ae9edc
Removed the regex requirement 2022-06-13 19:13:41 +02:00
Vicky ffb8479e53
Merge pull request #4 from idle-org/features/copy
Features/copy
2022-06-13 18:00:45 +02:00
Vicky 650229af5c
Merge pull request #3 from idle-org/features/bugfix-error-mngr
Fixed weird code duplication
2022-06-13 18:00:18 +02:00
Maximilien Rigaut 3abb5c53f7
Added copy behaviour to config file 2022-06-12 21:00:10 +02:00
Maximilien Rigaut c8082c44a3
Fixed weird code duplication 2022-06-12 14:14:40 +02:00
Maximilien Rigaut 4eb9247c5f
Test copy feature 2022-06-12 14:07:17 +02:00
Maximilien Rigaut e9837dbf99
Copy feature 2022-06-12 14:07:03 +02:00
Vicky cf8c0d56d4
Update README.md
Added template customisation objective for v3
2022-06-10 10:02:50 +02:00
Maximilien Rigaut 62666e0ab4
Merge remote-tracking branch 'origin/main' into develop 2022-06-09 19:27:21 +02:00
Maximilien Rigaut 1e7eea3466
Merge remote-tracking branch 'origin/releases/1.0.1' into develop
# Conflicts:
#	src/bootstraparse/modules/sitecreator.py
2022-06-09 19:23:56 +02:00
vicky b57ca251a0
minor code refactor 2022-06-09 18:58:00 +02:00
Vicky c8b60a7be0
Merge pull request #2 from idle-org/feature/doc
Feature/doc
2022-06-09 18:54:41 +02:00
Maximilien Rigaut 0e86cc5ca4
syntax.py : Doc and rtypes 2022-06-09 17:16:41 +02:00
Maximilien Rigaut ef9b53fcbc
sitecreator.py : Doc and rtypes 2022-06-09 17:16:34 +02:00
Maximilien Rigaut 6100f716b5
Test doc 2022-06-09 17:16:23 +02:00
Maximilien Rigaut dfc8a7b2d2
sitecrawler.py : Doc and rtypes 2022-06-09 16:57:44 +02:00
Maximilien Rigaut 00786d8480
Continuous integration of list 2022-06-09 16:41:58 +02:00
Maximilien Rigaut d1f4eaa6f2
preparser.py : Doc and rtypes 2022-06-09 16:36:16 +02:00
Maximilien Rigaut c52b7c959f
Typos 2022-06-09 16:35:26 +02:00
Maximilien Rigaut 1fa28a41a7
Added __init__.py 2022-06-09 16:35:04 +02:00
Maximilien Rigaut c4ee5f1e43
pathresolver.py : rtype and function doc (part) 2022-06-09 15:22:09 +02:00
Maximilien Rigaut 43db9f7a09
export.py : rtype and function doc (part) 2022-06-09 15:21:56 +02:00
Maximilien Rigaut e64c6d7f41
error_mngr.py : rtype and function doc (part) 2022-06-09 15:21:46 +02:00
Maximilien Rigaut df2e0941c8
environment.py : rtype and function doc (part) 2022-06-09 15:21:24 +02:00
Vicky 8641fa8d9d
Merge pull request #1 from idle-org/hotfix
Hotfix
2022-06-09 11:39:49 +02:00
vicky 62ad85b8f4
moved dict_check and str_dict_check from error_mngr to tools 2022-06-09 11:25:08 +02:00
Vicky 6cd7f8047a
Update config.py
changed log_message level to info
2022-06-09 01:03:31 +02:00
vicky ca773426db
changed warning to info in overwriting confing 2022-06-09 00:11:51 +02:00
Vicky e429175886
Update README.md
Added a bullet point for v1.0.2
2022-06-08 22:03:09 +02:00
Vicky 3f7ef8a922
Update index.bpr
Corrected [] and {}
2022-06-08 21:33:41 +02:00
Maximilien Rigaut 5a2c044dc1
context_mngr.py : rtype and function doc (part) 2022-06-08 20:19:05 +02:00
Maximilien Rigaut 7ab6f02635
PEP 2022-06-08 19:47:08 +02:00
Maximilien Rigaut fbc899cb2c
config: rtype and function doc 2022-06-08 19:47:08 +02:00
vicky bb5bec7a57
initial commit for release 2022-06-08 00:06:22 +02:00
37 changed files with 1049 additions and 522 deletions

37
.github/workflows/python-deploy.yml vendored Normal file
View File

@ -0,0 +1,37 @@
# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: Python Deployment
on:
push:
branches: [ main, test*]
pull_request:
branches: [ main, test* ]
jobs:
deployment:
runs-on: ubuntu-latest
environment: production
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Build the application
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
pip install -e .
pip install pydoctor
python -m bootstraparse "example_userfiles" "docs"
- name: Build the documentation
run: |
pydoctor --make-html --html-output=docs/apidoc src/bootstraparse/
- name: deploy
uses: JamesIves/github-pages-deploy-action@v4.3.3
with:
branch: gh-pages # The branch the action should deploy to.
folder: docs

View File

@ -1,13 +1,13 @@
# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: Python application
name: Python Tests and Lint
on:
push:
branches: [ main, test ]
branches: [ main, release, releases/*, test*]
pull_request:
branches: [ main, test ]
branches: [ main, release, releases/*, test* ]
jobs:
@ -40,23 +40,3 @@ jobs:
- name: Test with pytest
run: |
pytest --cov-fail-under=100 -o xfail_strict=true --cov-report=term-missing .
deployment:
runs-on: ubuntu-latest
environment: production
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Build the application
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
pip install -e .
python -m bootstraparse "example_userfiles" "docs"
- name: deploy
uses: JamesIves/github-pages-deploy-action@v4.3.3
with:
branch: gh-pages # The branch the action should deploy to.
folder: docs

3
.gitignore vendored
View File

@ -156,4 +156,5 @@ cython_debug/
example_userfiles/output/
example_userfiles/output/*
dev_outputs/*
/example_output/*
example_output/*
docs/

View File

@ -5,39 +5,70 @@ bootstraparse aims to be customisable in that regard, but its first iteration an
You can refer to the `example_userfiles` for an idea of what the parser expects you to feed it and find an example of our automated generation at https://idle-org.github.io/bootstraparse/ for this particular folder. Or you can run the program yourself for the same result!
[![Python application](https://github.com/idle-org/bootstraparse/actions/workflows/python-app.yml/badge.svg?branch=main)](https://github.com/idle-org/bootstraparse/actions/workflows/python-app.yml)
- Main
- [![Python Tests and Lint](https://github.com/idle-org/bootstraparse/actions/workflows/python-tests.yml/badge.svg?branch=main)](https://github.com/idle-org/bootstraparse/actions/workflows/python-tests.yml)
- Releases
- [![Python Tests and Lint](https://github.com/idle-org/bootstraparse/actions/workflows/python-tests.yml/badge.svg?branch=develop)](https://github.com/idle-org/bootstraparse/actions/workflows/python-tests.yml)
- Deploy
- [![Python Deployment](https://github.com/idle-org/bootstraparse/actions/workflows/python-deploy.yml/badge.svg?branch=main)](https://github.com/idle-org/bootstraparse/actions/workflows/python-deploy.yml)
---
## Roadmap
## Release Notes
### V1.0.1
- Make the default example site way more useful
- Remove old regex usage
- Add copy behaviour to config file
- Add the type and return types of all functions
- Add documentation to remaining lone functions
- Write a documentation for the application (use specs.yaml)
- Copy un-parsable files to destination folder without modifications
- Many bug fixes and improvements
### V1.0.2
- Put out many fires
### V1.0.3
- Copy un-parsable files to destination folder without modifications
- Add copy behaviour to config file
- Put out many more fires
- Fixed continuous integration
- Added doc generation
### V1.0.4
- Made the default example site more useful
- Wrote a barebone documentation for the application
## Roadmap
### V1.0.5
- Add more context to the warning messages ☐
- Test configs more in-depth ☐
- Test overall site generation ☐
- Move the WARNING-level output to INFO for config overwrite ☐
### V1.0.6
- Option to ignore br behavior ☐
- Make the default example site way more useful ☐
### V1.1
- Functioning `table` Token
- Functioning `code` Token
- Establish a list of all configurable parameters to implement in the future, and update the roadmap with them.
- Decide level of logic to be implemented, and whether it should be configurable
- Update the uses of remaining files
- Functioning `table` Token ☐
- Functioning `code` Token ☐
- Establish a list of all configurable parameters to implement in the future, and update the roadmap with them. ☐
- Decide level of logic to be implemented, and whether it should be configurable ☐
- Update the uses of remaining files ☐
- Add a global line count and a line tracker to the parser ☐
### V1.2
- Check error messages and add a real debug level to parameters
- Functioning `blockquote` Token
- Remove old regex usage
- Check error messages and add a real debug level to parameters ☐
- Thouroughly test logging, and error messages for the user ☐
- Functioning `blockquote` Token ☐
### V1.3
- Add advanced lookahead logic for `*`
- Functioning `lead` Token
### V1.4
- Add html indentation for human readability of the output.
- Add html indentation for human readability of the output. ☐
- Write thourough documentation for the application (use specs.yaml) ☐
### V2
- Add a functioning post-context enhancer able to generate menus and elements from arbitrary logic.
- Achieve perfect markdown compatibility with appropriate config parameters
### V3
- Add Template customisation.

View File

@ -10,7 +10,7 @@ shortcuts:
tab_bnw: 'tab_bnw_imported'
tab_color: 'tab_colour_imported'
custom_permutation: 'custom_permutated'
empty: ''
start: |
<!doctype html>
<html lang="en">

View File

@ -0,0 +1,48 @@
Structural Elements:
<<div
This is a div
div>>{class='blue', 123}{{cinsertion}}[var='test', number=11]
<<article
This is an article
article>>{class='blue', 123}{{cinsertion}}[var='test', number=11]
<<aside
This is an aside
aside>>{class='blue', 123}{{cinsertion}}[var='test', number=11]
<<section
This is a section
section>>{class='blue', 123}{{cinsertion}}[var='test', number=11]
<<div
<<article
<<aside
<<section
Text
section>>{class='blue', 123}{{cinsertion}}[var='test', number=11]
aside>>{class='blue', 123}{{cinsertion}}[var='test', number=11]
article>>{class='blue', 123}{{cinsertion}}[var='test', number=11]
div>>{class='blue', 123}{{cinsertion}}[var='test', number=11]
# Header 1 #{class='blue', 123}{{cinsertion}}[var='test', number=11]
! Display 1 !{class='blue', 123}{{cinsertion}}[var='test', number=11]
Inline Element:
[Link text]('http://this.is.a.link')
*This is in italic*
**This is strong**
~~This is stricken through~~
__This is underlined__
(#1)This is a custom span(#1)
Oneline elements:
- This is an unordered list item
- This is an unordered list item
- This is an unordered list item{class='blue', 123}{{cinsertion}}[var='test', number=11]
#. This is an ordered list item
#. This is an ordered list item
#. This is an ordered list item{class='blue', 123}{{cinsertion}}[var='test', number=11]
This is a long text segment. This is a long text segment. This is a long text segment. This is a long text segment. This is a long text segment.
This is still the same long text segment. This is still the same long text segment. This is still the same long text segment.
Use a double linebreak to force linebreaks.

View File

@ -1,49 +1,28 @@
@[start]['bootstrap']
Structural Elements:
@[start]['Bootstraparse']
<<div
This is a div
div>>[class='blue', 123]{var='test', number=11}{{cinsertion}}
<<article
This is an article
article>>[class='blue', 123]{var='test', number=11}{{cinsertion}}
<<aside
This is an aside
aside>>[class='blue', 123]{var='test', number=11}{{cinsertion}}
<<section
This is a section
section>>[class='blue', 123]{var='test', number=11}{{cinsertion}}
<<header
<<div
<<article
<<aside
<<section
Text
section>>[class='blue', 123]{var='test', number=11}{{cinsertion}}
aside>>[class='blue', 123]{var='test', number=11}{{cinsertion}}
article>>[class='blue', 123]{var='test', number=11}{{cinsertion}}
div>>[class='blue', 123]{var='test', number=11}{{cinsertion}}
# Header 1 #[class='blue', 123]{var='test', number=11}{{cinsertion}}
! Display 1 ![class='blue', 123]{var='test', number=11}{{cinsertion}}
<<div
Bootstraparse
div>>{{blog-header-logo text-dark text-center border-bottom}}
div>>{{row flex-nowrap justify-content-between align-items-center}}
header>>{{blog-header py-3}}
<<div
# What is bootstraparse ? #
bootstraparse is a tool that allows you to bootstrap your own project.
In a very small number of steps, you can create a new project with the
following features:
- Imports, with the <code> ::&lt; filename > </code>syntax you can import another file and have it's contents dumped into your file.
- Markup, a subset of markdown is already supported and the rest is incomming !
- Aliases : With the <code> @&#91;alias_name&#93; </code> syntax you can create an alias for a string of text, with replacement capabilities.
- Images: With the <code> @&#123;imagename&#125; </code> syntax you can specify an image template to import, with replacement capabilities.
And much much more !
Inline Element:
[Link text]('http://this.is.a.link')
*This is in italic*
**This is strong**
~~This is stricken through~~
__This is underlined__
(#1)This is a custom span(#1)
Oneline elements:
- This is an unordered list item
- This is an unordered list item
- This is an unordered list item[class='blue', 123]{var='test', number=11}{{cinsertion}}
#. This is an ordered list item
#. This is an ordered list item
#. This is an ordered list item[class='blue', 123]{var='test', number=11}{{cinsertion}}
This is a long text segment. This is a long text segment. This is a long text segment. This is a long text segment. This is a long text segment. This is a long text segment. This is a long text segment. This is a long text segment.
I wish there could be linebreaks.
We encourage you to read the [documentation]("apidoc/index.html") to learn more about bootstraparse
or explore the [source code]("https://github.com/idle-org/bootstraparse")
div>>{{nav-scroller py-1 mb-2}}
div>>{{container}}
@[end]

View File

@ -1,7 +1,5 @@
pyyaml==6.0
rich==12.0.1
regex==2022.3.15
pyparse~=1.1.7
pyparsing[diagrams]~=3.0.7
pytest==7.1.1

View File

@ -5,4 +5,3 @@ pytest-cov==3.0.0
mypy==0.941
pyyaml==6.0
rich==12.0.1
regex==2022.3.15

View File

@ -2,15 +2,14 @@
name = bootstraparse
description = A custom markdown parser for Bootstrap & Python
author = Vkyfox
version = 1.0
version = 1.0.4
license = Mozilla Public License Version 2.0
license_file = LICENSE
platforms = unix, linux, osx, cygwin, win32
classifiers =
Programming Language :: Python :: 3
Programming Language :: Python :: 3 :: Only
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.10
Programming Language :: Python :: 3.9
[options]
@ -19,7 +18,7 @@ packages =
bootstraparse.modules
install_requires =
pyyaml>=6.0
python_requires = >=3.6
python_requires = >=3.9
package_dir =
=src

View File

@ -0,0 +1,5 @@
"""
Bootstraparse is a tool to parse a folder and create a website from it.
After installing it with pip, you can use it by running the following command:
- bootstraparse <origin> <destination>
"""

View File

@ -1,8 +1,8 @@
#!/usr/bin/python3
# Main program, use this to start parsing
""" Main program, use this to start parsing"""
from bootstraparse.modules import sitecreator
from bootstraparse.modules import sitecreator, error_mngr
import argparse
import sys
@ -21,5 +21,6 @@ def parse(_args):
if __name__ == "__main__": # pragma: no cover
args = parse(sys.argv[1:])
error_mngr.init_logging(filename=None, loglevel="DEBUG", filemode='w', handler=None)
if sitecreator.create_website(args.origin, args.destination) == 0:
print("Bootstraparse run successful!")

View File

@ -2,7 +2,7 @@ global:
template: "bootstrap.yml"
output: "example_site/"
bootstrap_version: "3.3.7"
version: "1.0.0"
version: "1.0.4"
bootstrap_location: ""
logging:
@ -21,3 +21,4 @@ export:
intermediate_files: true
type: "html"
force_rewrite: true
copy_unparsable_files: copy

View File

@ -0,0 +1,3 @@
"""
The modules in this folder are used to parse the website.
"""

View File

@ -1,11 +1,11 @@
# Interprets config files
# The Config will be loaded by the environment at runtime and should be accessed that way
# Usage:
# from bootstraparse.modules.config import ConfigLoader
# config = ConfigLoader([list of config files], extensions=[list of extensions])
# config['file']['key']
# The ConfigLoader class has load_from_file and load_from_folder methods and __getitem__ for accessing loaded elements
"""Interprets config files
The Config will be loaded by the environment at runtime and should be accessed that way
Usage:
- from bootstraparse.modules.config import ConfigLoader
- config = ConfigLoader([list of config files], extensions=[list of extensions])
- config['file']['key']
The ConfigLoader class has load_from_file and load_from_folder methods and __getitem__ for accessing loaded elements
"""
import os
import rich
@ -17,16 +17,16 @@ class ConfigLoader:
"""
Reads all config files in a folder
Behaves like a dictionary of all config files
return FileNotFound Error on Error
:param config_folder: The folder to read
"""
def __init__(self, config_folder=None, extensions=("yaml", "yml")):
"""
Defines the config folder and loads all configs
:param config_folder: path to config file
:type config_folder: (str | list[str])
:param extensions: file extensions to load
:return: None
:type extensions: list[str]
:raise TypeError: if config_folder is not a string or a list of strings
"""
if config_folder is None:
self.config_folders = []
@ -54,6 +54,7 @@ class ConfigLoader:
"""
Add a Folder to the list of configs
:param folder: path to folder
:type folder: str
:return: None
"""
self.config_folders.append(folder)
@ -63,6 +64,7 @@ class ConfigLoader:
"""
Loads a config file
:param filepath: path to config file
:type filepath: str
:return: None
"""
basename = os.path.basename(filepath)
@ -72,15 +74,22 @@ class ConfigLoader:
if name not in self.loaded_conf:
self.loaded_conf[name] = yaml.safe_load(f)
else:
self.loaded_conf[name].update(yaml.safe_load(f))
error_mngr.log_message(f"Warning: {name} is already in {self.loaded_conf}", level='WARNING')
new_config = yaml.safe_load(f)
for key in new_config:
if key in self.loaded_conf[name]:
self.loaded_conf[name][key].update(new_config[key])
else:
self.loaded_conf[name][key] = new_config[key]
error_mngr.log_message(f"Warning: {name} is already in {self.loaded_conf}", level='CRITICAL')
except BaseException as e:
error_mngr.log_message(f'Error parsing in file {basename} at {filepath}.', level='CRITICAL')
error_mngr.log_exception(str(e), level='CRITICAL')
error_mngr.log_exception(e, level='CRITICAL')
def load_from_folder(self, folder):
"""
Loads all configs in the config folder
:param folder: path to folder
:type folder: str
:return: None
"""
for filename in os.listdir(folder):
@ -92,6 +101,8 @@ class ConfigLoader:
"""
Returns a yaml config object if in self.loaded_conf
:param item: config key
:type item: str
:raise KeyError: if item is not in self.loaded_conf
:return: config value
"""
try:
@ -102,36 +113,21 @@ class ConfigLoader:
def __contains__(self, item):
"""
Same as a dictionary.
:param item: config key
:type item: str
:return: True if item is in self.loaded_conf
"""
return item in self.loaded_conf
def __repr__(self):
"""
Returns config as string
:return: config as string
:return: the dict of configs and their values
:rtype: str
"""
return self.loaded_conf.__repr__()
return str(self.loaded_conf.__repr__())
# class UserConfig(ConfigLoader):
# """
# Reads user config file.
# :param config_folder: path to config file
# :return: None
# """
# def __init__(self, config_folder):
# super().__init__(config_folder)
#
#
# class GlobalConfig(ConfigLoader):
# """
# Reads global config file.
# :param config_folder: path to config file
# :return: None
# """
# def __init__(self, config_folder):
# super().__init__(config_folder)
if __name__ == '__main__': # pragma: no cover
conf = ConfigLoader("../configs/")
conf.add_folder("../../../example_userfiles/config/")

View File

@ -1,17 +1,16 @@
# Interprets where the parser is located and resolves errors
# You're probably here for the ContextManager class.
# Containers have class_name, debug_map and validate methods for debug,
# self.content: accessible via __getitem__, __setitem__, add, __len__, __iter__ and __getslice__.
# self.optionals: accessible via __rshift__ (remapped to return self.map[other]())
# and self.map: accessible via __invert__ (remapped to return self.optionals)
# Usage:
# from bootstraparse.modules.context_mngr import ContextManager
# ctx = ContextManager()
# ctx.???
# container = BaseContainer()
# container[number] -> The number element in the content
# "class_insert" >> container[number] -> Get an element from one of the mapped methods
"""Interprets where the parser is located and resolves errors
You're probably here for the ContextManager class.
Containers have class_name, debug_map and validate methods for debug,
- self.content: accessible via __getitem__, __setitem__, add, __len__, __iter__ and __getslice__.
- self.optionals: accessible via __rshift__ (remapped to return self.map[other]())
- and self.map: accessible via __invert__ (remapped to return self.optionals)
Usage:
- from bootstraparse.modules.context_mngr import ContextManager
- ctx = ContextManager()
- container = BaseContainer()
- container[number] -> The number element in the content
- "class_insert" >> container[number] -> Get an element from one of the mapped methods
"""
from bootstraparse.modules import syntax, error_mngr, export
from bootstraparse.modules.error_mngr import MismatchedContainerError, log_exception, log_message, LonelyOptionalError # noqa
@ -24,6 +23,14 @@ class BaseContainer:
subtype = None
def __init__(self, content=None, optionals=None, others=None):
"""
:param content: List of tokens to be added to the container
:param optionals: List of optional tokens to be added to the container (from the parser)
:param others: List of values to add to the .format() method
:type content: (list[(syntax.SemanticType | BaseContainer)])
:type optionals: (syntax.SemanticType)
:type others: (dict[str, str])
"""
if content is None:
content = []
if others is None:
@ -34,24 +41,41 @@ class BaseContainer:
self.indentation_level = 0 # future: add an indentation level to every token for human readability of the final output
def get_content(self, exm, arbitrary_list=None):
"""
Get the content of any BaseContainer object (default self.content).
:param exm: ExportManager to use for exporting
:param arbitrary_list: List to get the content from
:type exm: export.ExportManager
:type arbitrary_list: list[BaseContainer]
:rtype : str
"""
output = ""
if not arbitrary_list:
arbitrary_list = self.content
# rich.inspect(arbitrary_list)
for element in arbitrary_list:
if isinstance(element, BaseContainer):
output += element.export(exm) + " "
# rich.inspect(output[:-1])
return output[:-1]
def get_optionals(self):
"""
Get the optionals of the container.
:rtype : (syntax.SemanticType | BaseContainer)
:return: The optionals of the container
"""
return self.optionals
def get_others(self):
"""
Get the other values of the container.
:rtype : dict[str, str]
:return: The other values of the container
"""
return self.others
def export(self, exm):
"""
Export the container to a string.
:type exm: export.ExportManager
:rtype : str
"""
@ -62,15 +86,27 @@ class BaseContainer:
return output
def add(self, other):
"""
Adds a list of elements to the container.
:param other: List of elements to add
:type other: syntax.SemanticType
:return: None
"""
self.content.append(other)
def class_name(self):
"""
A string representation of the class name.
:rtype : str
"""
return self.__class__.__name__
def validate(self, other):
"""
Takes a list as input and checks if every element is in map.
:return: Bool
:param other: List of elements to check
:type other: list[(syntax.SemanticType | BaseContainer)]
:return: True if every element is in map, False otherwise
"""
for o in other:
if o not in self.map:
@ -78,30 +114,68 @@ class BaseContainer:
return True
def debug_map(self):
"""
Prints the map of the container.
:return: None
"""
print(f'Debug for {self.class_name()} <{id(self)}>')
for k, v in self.map.items():
print(f'{k} = {v}')
def __len__(self):
"""
:rtype : int
"""
return len(self.content)
def __iter__(self):
"""
:ytype : iter[syntax.SemanticType]
"""
for content in self.content:
yield content
def __getitem__(self, item):
"""
:param item: Index of the element to get
:type item: int
:rtype : (syntax.SemanticType | BaseContainer)
"""
return self.content[item]
def __setitem__(self, key, value):
"""
:param key: Index of the element to set
:param value: Value to set
:type key: int
:type value: (syntax.SemanticType | BaseContainer)
"""
self.content[key] = value
def __getslice__(self, start, end):
"""
:param start: Start index of the slice
:param end: End index of the slice
:type start: int
:type end: int
:rtype : list[(syntax.SemanticType | BaseContainer)]
"""
return self.content[start:end]
def __rshift__(self, other):
"""
:param other: The name of the map element to get
:type other: str
:rtype : (syntax.SemanticType | BaseContainer)
"""
return self.map[other]()
def __eq__(self, other):
"""
:param other: The other object to compare to
:type other: BaseContainer | list[(syntax.SemanticType | BaseContainer)] | syntax.SemanticType
:rtype : bool
"""
if not hasattr(other, "__len__"):
return False
if len(self) != len(other):
@ -123,13 +197,23 @@ class BaseContainer:
return True
def __invert__(self):
"""
Returns the inverted optionals of the container.
:rtype : BaseContainer
"""
return self.get_optionals()
def __str__(self):
"""
:rtype : str
"""
return '{} ({})'.format(", ".join(map(str, self.content)),
", ".join(map(str, self.get_optionals())) if self.get_optionals() else "")
", ".join(map(str, self.get_optionals().content)) if self.get_optionals() else "")
def __repr__(self):
"""
:rtype : str
"""
representation = {}
for content in self.content:
if type(content) in representation:
@ -146,6 +230,9 @@ class BaseContainer:
)
def print_all(self, ident=''):
"""
Prints the container and all its content.
"""
print(ident + self.class_name())
for e in self:
if isinstance(e, BaseContainer):
@ -154,8 +241,12 @@ class BaseContainer:
print(ident + "> " + str(e))
print("")
def to_container(self, filter_func=None): # idea: additional parameters for encapsulation
return self
def to_container(self, filter_func=None): # noqa : C901
"""
Returns a container with the same content as the current container.
Used for compatibility with the syntax for encapsulation.
"""
return self # idea: additional parameters for encapsulation
# Define containers all the Enhanced text elements, divs, headers, list and any element that can be a container
@ -194,7 +285,7 @@ class EtCustomSpanContainer(BaseContainer):
type = "inline_elements"
def export(self, exm):
self.subtype = "custom_" + self.content[0].content[0]
self.subtype = "custom_" + self.content[0].content[0] # noqa F821 (self.content[0] is a token, by definition
return super().export(exm)
@ -244,7 +335,7 @@ class SeContainer(BaseContainer):
type = "structural_elements"
def export(self, exm):
self.subtype = self[0].content[0]
self.subtype = "se_"+self[0].content[0]
return super().export(exm)
@ -253,7 +344,7 @@ class HeaderContainer(BaseContainer):
subtype = "header"
def export(self, exm):
self.others = {}
self.others = {} # noqa F821
self.others["header_level"] = len(self.content[0].content[0])
return super().export(exm)
@ -345,10 +436,12 @@ class ContextManager:
def __init__(self, parsed_list, name=None, ident=0):
"""
Takes a list of parsed tokens.
Parameters
----------
parsed_list : list[syntax.SemanticType]
List of parsed tokens output by our parser.
:type parsed_list : list[syntax.SemanticType]
:param parsed_list: List of parsed tokens output by our parser.
:type name: str
:param name: Name of the file being parsed.
:type ident: int
:param ident: Number of spaces to indent the output.
"""
self.output = []
self.parsed_list = parsed_list
@ -369,15 +462,12 @@ class ContextManager:
"""
Method to encapsulate a number of tokens together as a final container object
and replace all elements in the pile with None, except the first one which becomes the container.
Parameters
----------
start : int
Index of the pile where the encapsulation must begin.
end : int
Index of the pile where the encapsulation must end.
Returns
-------
list[(BaseContainer|None)]
:type start : int
:param start: Index of the pile where the encapsulation must begin.
:type end: int
:param end: Index of the pile where the encapsulation must end.
:rtype: list[(BaseContainer|None)]
:return: List of containers and None objects.
"""
# Enabling this will allow us to debug the pile
# print(f"Step : [{end}] - encapsulation:")
@ -419,25 +509,38 @@ class ContextManager:
container.add(self.pile[end])
self.pile[end] = None
self.pile[start] = container
return self.pile # ? I think this is the right way to return the pile
# print(" <<< Pile:", self.pile)
def _add_matched(self, label, index):
"""
Method to add a matched element to the dictionary of matched elements.
:type label : str
:param label: Label of the matched element.
:type index: int
:param index: Index of the matched element in the pile.
"""
if label not in self.matched_elements:
self.matched_elements[label] = []
self.matched_elements[label] += [index]
def _get_matched(self, label):
"""
Method to get the index of the last matched element of a given label.
:type label: str
:param label: Label of the matched element.
:rtype: int
:return: Index of the last matched element
"""
return self.matched_elements[label].pop()
def __call__(self):
"""
Interprets the list of tokens provided and deduces context, encapsulating them to their closest neighbour and
containing all tokens inbetween.
Returns
-------
list[BaseContainer]
Returns the pile entirely processed as a list of containers.
:rtype: list[BaseContainer]
:return: Returns the pile entirely processed as a list of containers.
:raises: MismatchedContainerError if a container is not matched to its corresponding token.
"""
# self.pile = self.parsed_list.copy()
if self.contextualised:
@ -448,6 +551,7 @@ class ContextManager:
while index < len(self.parsed_list):
token = self.parsed_list[index]
token.line_number = line_number
token.index = index
token.file_name = self.name
token.ident = self.ident
self.pile.append(token)
@ -480,16 +584,17 @@ class ContextManager:
# Error if closing token does not have a start
elif isinstance(token, syntax.ClosedSemanticType):
raise MismatchedContainerError(token)
raise MismatchedContainerError(token, self.parsed_list)
# Starting token by default (can cause unintended behaviours on bad implementations)
elif isinstance(token, syntax.TokensToMatch):
self._add_matched(token.label, index)
else:
raise MismatchedContainerError(token)
raise MismatchedContainerError(token, self.parsed_list)
except MismatchedContainerError as e:
error_mngr.log_message(f"In context: {e.get_context()}", level="CRITICAL")
error_mngr.log_exception(e, level="CRITICAL") # FUTURE: Try to guess some hints.
index += 1
@ -497,6 +602,10 @@ class ContextManager:
return self.finalize_pile()
def __iter__(self):
"""
Iterator over the pile.
:ytype: BaseContainer
"""
for e in self.pile:
if e:
yield e
@ -505,6 +614,7 @@ class ContextManager:
"""
Function for cleaning up of the pile after full contextualisation.
Removes Nones and checks for any errors or illogical containers.
:return: list[BaseContainer]
"""
final_pile = []
for p in self.pile:
@ -532,6 +642,12 @@ class ContextManager:
def lookahead(self, token, index):
"""
Iterates the pile beginning from index and looks for all tokens matching labels with token.
:type token : syntax.BaseContainer
:param token: Token to look for.
:type index : int
:param index: Index of the token in the pile.
:return: (number of tokens matched, number of linebreaks)
:rtype: (int, int)
"""
range_to_encapsulate = 0
line_skipped = 0
@ -557,18 +673,23 @@ class ContextManager:
return range_to_encapsulate, line_skipped
def recontext(self, token):
"""
Function to recontextualise the content of a token.
:param token: Token to recontextualise.
:type token: syntax.SemanticType
:return: Token to recontextualise.
:rtype: syntax.SemanticType
"""
token.content = ContextManager(token.content, name=self.name)()
def get_last_container_in_pile(self, index):
"""
Returns the last element in the pile if it is a Container, raises an error otherwise.
Return
------
BaseContainer
Raises
------
error_mngr.LonelyOptionalError
:type index : int
:param index: Index of the element to check.
:return: Last element in the pile if it is a Container, raises an error otherwise.
:rtype: BaseContainer
:raise: error_mngr.LonelyOptionalError when an element is found in the pile that is not encapsulated in a container.
"""
i = index-1 # skip last token as it is self
while i >= 0:
@ -581,6 +702,9 @@ class ContextManager:
log_exception(LonelyOptionalError(self.pile[index], None), level="CRITICAL")
def print_all(self):
"""
Prints the pile.
"""
for e in self:
if isinstance(e, BaseContainer):
e.print_all()

View File

@ -1,19 +1,20 @@
# Module for containing super important variables
# Environment has self._mParams for mandatory parameters, self._sParams for secondary parameters and
# self.wasInitialised and integrity_check for debug.
# __getattr__ and __setattr__ check for existing super, s or m parameters, returns error if not found.
# Usage:
# from bootstraparse.modules.environment import Environment
# env = Environment()
# env.integrity_check() -> Checks if all mandatory parameters are set.
# env["config"] -> returns config object
# env["template"] -> returns template object
# env["export_mngr"] -> returns export manager object
# env["global_path"] -> returns global path ???
# env["user_path"] -> returns user path ???
# env["site_path"] -> returns site path
# env["export"] -> returns export object ???
# env["site_crawler"] -> returns site crawler object
"""Module for containing super important variables
Environment has self._mParams for mandatory parameters, self._sParams for secondary parameters and
- self.wasInitialised and integrity_check for debug.
- __getattr__ and __setattr__ check for existing super, s or m parameters, returns error if not found.
Usage:
- from bootstraparse.modules.environment import Environment
- env = Environment()
- env.integrity_check() -> Checks if all mandatory parameters are set.
- env["config"] -> returns config object
- env["template"] -> returns template object
- env["export_mngr"] -> returns export manager object
- env["global_path"] -> returns global path ???
- env["user_path"] -> returns user path ???
- env["site_path"] -> returns site path
- env["export"] -> returns export object ???
- env["site_crawler"] -> returns site crawler object
"""
from bootstraparse.modules import error_mngr
@ -26,6 +27,10 @@ class Environment:
"""
def __init__(self):
"""
Initialises the environment, all parameters are set to None.
Set _wasInitialised to False for all parameters.
"""
# mandatoryParameters
self._mParams = {
'config': None,
@ -60,8 +65,10 @@ class Environment:
def __getattr__(self, attribute):
"""
Getter for all parameters.
Getter for all parameters, looks for super if the parameter is prefaced with an underscore.
Looks for mandatory parameters first, then secondary parameters.
:param attribute: The name of the parameter to get.
:type attribute: str
:return: The value of the parameter.
"""
if attribute[0] == "_":
@ -80,8 +87,9 @@ class Environment:
def __setattr__(self, attribute, value):
"""
Setter for all parameters.
Setter for all parameters, looks for super if the parameter is prefaced with an underscore.
:param attribute: The name of the parameter to set.
:type attribute: str
"""
if attribute[0] == "_":
super().__setattr__(attribute, value)

View File

@ -1,12 +1,14 @@
# Dedicated module for defining and interpreting errors as well as generating error messages
# You're probably here for log_message or log_exception.
# They take: message, level=("ERROR, "INFO", "WARNING", "DEBUG", "CRITICAL")
# There's also a dict_check(dic, *args) that returns true if all args are keys in the dict.
# Usage:
# from bootstraparse.modules.error_mngr import log_message, log_exception
# log_message("This is a message", level="INFO") level=("ERROR, "INFO", "WARNING", "DEBUG", "CRITICAL")
# log_exception(Exception("This is an exception"), level="ERROR")
# dict_check({"a": 1, "b": 2}, "a", "b") # returns [True, True]
"""
Dedicated module for defining and interpreting errors as well as generating error messages
You're probably here for log_message or log_exception.
- They take: message, level=("ERROR, "INFO", "WARNING", "DEBUG", "CRITICAL")
There's also a dict_check(dic, *args) that returns true if all args are keys in the dict.
Usage:
- from bootstraparse.modules.error_mngr import log_message, log_exception
- log_message("This is a message", level="INFO") level=("ERROR, "INFO", "WARNING", "DEBUG", "CRITICAL")
- log_exception(Exception("This is an exception"), level="ERROR")
- dict_check({"a": 1, "b": 2}, "a", "b") # returns [True, True]
"""
import logging
import traceback
@ -15,17 +17,19 @@ from bootstraparse.modules.tools import __GLk # __GFi, __GFu, __GL
# Define the error codes
_ERRORS = ["ParsingError", "MismatchedContainerError"]
__all__ = _ERRORS+[]
def init_logging(filename=None, loglevel="ERROR", filemode='w', handler=None):
"""
Initializes logging
:param filename: The name of the log file
:param loglevel: The level of logging
:type loglevel: str
:param filemode: The mode of the log file
:param filename: The path for the log file
:param loglevel: The level of logging: "ERROR", "INFO", "WARNING", "DEBUG", "CRITICAL"
:param filemode: The mode of the log file: "w", "a"
:param handler: The handler to use
:type filename: str
:type loglevel: str
:type filemode: str
:type handler: logging.Handler
:return: None
"""
loglevel = loglevel.upper()
@ -50,6 +54,7 @@ def log_message(message, level="ERROR"):
:param message: The message to log
:param level: The level of the message
:type level: str
:type message: str
:return: None
"""
level = level.lower()
@ -83,52 +88,79 @@ def log_exception(exception, level="ERROR"):
return
def dict_check(dic, *args):
"""
Helper function to test existence within a dictionary.
:param dic: The dictionary of dictionaries to check
:param args: Any number of keys to check the dictionaries for
:return: A table of Booleans for every key checked.
"""
output = [False for _ in args]
for i in range(len(args)):
if args[i] in dic:
output[i] = True
dic = dic[args[i]]
else:
break
return output
def str_dict_check(dic, *args):
"""
Helper function to test existence within a dictionary and return a string of the values.
'arg[0]' : True or False
"""
out = dict_check(dic, *args)
return "\n".join([f"{args[i]}: {out[i]}" for i in range(len(out))])
class BootstraparseError(Exception):
"""
Base class for all Bootstraparse errors
"""
pass
class BootstraparseTokenError(BootstraparseError):
"""
Error on a token, provides if possible the line and column number of the error,
the file name and the message of the error
and if possible a context for the error
"""
def __init__(self, token, pile, message):
self.token = token
self.pile = pile
if token is not None:
self.line = token.line_number
self.index = token.index
self.label = token.label
self.name = token.file_name
else:
self.line = None
self.index = None
self.label = None
self.name = None
self.context = self.get_context()
super().__init__(message+"\n"+self.context_multiline())
def get_context(self, context_size=5):
"""
Returns the context of the error
:return: The context of the error
:rtype: str
"""
if self.pile:
st = max(0, self.index - context_size)
en = min(len(self.pile), self.index + context_size)
return self.pile[st:en]
return []
def context_multiline(self, context_size=10):
"""
Returns the context of the error as a multiline string
:return: The context of the error
:rtype: str
"""
if self.pile:
return "\n".join([
f"{' ' if i!=context_size else '> '}{i-context_size:2d}: {j}"
for i, j in enumerate(self.get_context(context_size)) if j.label != "linebreak"
])
return ""
class ParsingError(Exception):
"""
Exception class for parsing errors
:param message: The message to display
:type message: str
:param line: The line number of the error
:param column: The column number of the error
:param file: The name of the file
:return: None
"""
def __init__(self, message=None, line=None, column=None, file=None):
"""
Initializes the ParsingError class with the filename and a line and column number
based on the current position in the file (if available)
:param message: The message to display
:type message: str
:param line: The line number of the error
:type line: int
:param column: The column number of the error
:type column: int
:param file: The name of the file
:type file: str
:return: None
"""
super().__init__(message)
self.line = line
@ -149,12 +181,18 @@ class ParsingError(Exception):
return str(self.message)
class MismatchedContainerError(BootstraparseError):
class MismatchedContainerError(BootstraparseTokenError):
"""
The token is not final and cannot be contained. Indicative of a mismatched token.
"""
def __init__(self, token):
def __init__(self, token, pile=None):
"""
Initializes the MismatchedContainerError class with the token that was not final
:param token: The token that is not final
:type token: syntax.SemanticType
"""
self.token = token
self.pile = pile
if token:
self.line = token.line_number
self.label = token.label
@ -163,7 +201,7 @@ class MismatchedContainerError(BootstraparseError):
self.line = None
self.label = None
self.name = None
super().__init__(f"Could not process {self.label} at line {self.line} in file {self.name}.")
super().__init__(token, pile, f"Could not process {self.label} at line {self.line} in file {self.name}.")
class LonelyOptionalError(BootstraparseError):
@ -171,6 +209,14 @@ class LonelyOptionalError(BootstraparseError):
Optional token could not be matched with last element in pile (not a container).
"""
def __init__(self, token, last_in_pile):
"""
Initializes the LonelyOptionalError class with the token found and the last element in the pile
Used when a token is found that is optional and cannot be matched with the last element in the pile (not a container)
:param token: The token that is not final
:type token: syntax.SemanticType
:param last_in_pile: The last element in the pile
:type last_in_pile: syntax.SemanticType
"""
self.token = token
if token:
self.line = token.line_number

View File

@ -1,16 +1,17 @@
# Module for final generation of the translated website
# Usage:
# from bootstraparse.modules.export import ExportManager, ExportRequest, ExportResponse
# rqs = ExportRequest("strucural_elements", "div", "optionals", {"id": "my_id", "class": "my_class"})
# rsp = ExportResponse("start_string", "end_string")
# em = ExportManager(config_file, template_file)
# em(ExportRequest()) -> ExportResponse()
# import rich
import rich
"""
Module for final generation of the translated website
Usage:
- from bootstraparse.modules.export import ExportManager, ExportRequest, ExportResponse
- rqs = ExportRequest("strucural_elements", "div", "optionals", {"id": "my_id", "class": "my_class"})
- rsp = ExportResponse("start_string", "end_string")
- em = ExportManager(config_file, template_file)
- em(ExportRequest()) -> ExportResponse()
"""
from io import StringIO
from bootstraparse.modules import config, pathresolver, error_mngr, context_mngr
from collections import namedtuple
from bootstraparse.modules import tools
from bootstraparse.modules import syntax # noqa F401
@ -47,7 +48,14 @@ class ExportManager:
"""
Transforms ExportRequest tuples to ExportResponse tuples with the config-provided appropriate markup.
"""
def __init__(self, cnoifg, templates): # noqa
def __init__(self, cnoifg, templates):
"""
Initialization function for the ExportManager.
:param cnoifg: Config file
:type cnoifg: config.ConfigLoader
:param templates: Templates file
:type templates: config.ConfigLoader
"""
self.config = cnoifg
self.templates = templates
self.advanced_export = {
@ -61,6 +69,12 @@ class ExportManager:
}
def __call__(self, export_request):
"""
Callable function for ExportManager, transforming ExportRequest tuples into ExportResponse tuples.
:param export_request: ExportRequest tuples
:type export_request: ExportRequest
:return: ExportResponse tuples
"""
return self.transform(export_request)
def transform(self, export_request):
@ -82,18 +96,10 @@ class ExportManager:
"""
Function for initializing other transform functions.
Parameters:
----------
export_request: ExportRequest tuples
Returns:
-------
start: str
Start string of the markup element
end: str
End string of the markup element
optionals: syntax.Optionals
Optionals object containing html_insert and class_insert
:param export_request: ExportRequest tuples
:type export_request: ExportRequest
:return: start, end, optionals
:rtype: str, str, str
"""
start, end = None, None
@ -101,8 +107,7 @@ class ExportManager:
start, end = self.templates["bootstrap"][export_request.type][export_request.subtype]
except KeyError:
log_entries = ["bootstrap", export_request.type, export_request.subtype]
log_ = error_mngr.dict_check(self.templates, *log_entries)
print()
log_ = tools.dict_check(self.templates, *log_entries)
error_mngr.log_exception(
KeyError(
f'Template "bootstrap"/{export_request.type}/{export_request.subtype} could not be found.\n' +
@ -151,6 +156,7 @@ class ExportManager:
"""
Specific template for display.
:type export_request: ExportRequest
:rtype: ExportResponse
"""
start, end, optionals = self._get_template(export_request)
start = start.format(optionals=optionals, display_level=export_request.others["display_level"])
@ -160,6 +166,7 @@ class ExportManager:
"""
Specific template for link.
:type export_request: ExportRequest
:rtype: ExportResponse
"""
start, end, _ = self._get_template(export_request)
start = start.format(url=export_request.others["url"])
@ -169,6 +176,7 @@ class ExportManager:
"""
Specific template for table content.
:type export_request: ExportRequest
:rtype: ExportResponse
"""
start, end, _ = self._get_template(export_request)
start = start.format(col_span=export_request.others["col_span"])
@ -178,6 +186,7 @@ class ExportManager:
"""
Specific template for images.
:type export_request: ExportRequest
:rtype: ExportResponse
"""
start, end, optionals = self._get_template(export_request)
end = end.format(optionals=optionals)
@ -191,12 +200,12 @@ class ContextConverter:
def __init__(self, pile, exporter, destination):
"""
Takes a pile and the exporter object
Parameters
----------
pile : list[context_mngr.BaseContainer]
pile output from the context manager.
exporter : ExportManager
Our ExportManager
:type pile : list[context_mngr.BaseContainer]
:param pile: output from the context manager.
:type exporter : ExportManager
:param exporter: Our ExportManager
:type destination : str
:param destination: Destination of the output file.
"""
self.io_output = StringIO()
self.pile = pile
@ -206,7 +215,8 @@ class ContextConverter:
def process_pile(self):
"""
Main method processing the given pile.
Processes the pile and writes the output to the io_output object.
:rtype: StringIO
"""
for container in self.pile:
self.io_output.write(container.export(self.exporter))
@ -224,6 +234,7 @@ class ContextConverter:
def readlines(self):
"""
Yields the converted pile line after line
:rtype: list[str]
"""
return self.io_output.readlines()
@ -232,6 +243,9 @@ class ContextConverter:
f"Status: {'initialized' if self.io_initialized else 'uninitialized'}"
def print_all(self):
"""
Prints the entirety of the converted pile
"""
print(self)
def __eq__(self, other):
@ -239,10 +253,11 @@ class ContextConverter:
if __name__ == '__main__': # pragma: no cover
import rich
from bootstraparse.modules import parser
io_string = StringIO(
"""[this is a link]('http://te.st')"""
"""! This is a test ! {{fgd}}"""
)
test = parser.parse_line(io_string)
__config = config.ConfigLoader(pathresolver.b_path("configs/"))

View File

@ -1,9 +1,10 @@
# Main parser, gets config files and parses them for the context manager.
# parse_line takes a io and outputs it all as a list of parsed elements
# Usage:
# from bootstraparse.modules.parser import parse_line
# parse_line(io) -> [element, element, element]
"""
Main parser, gets config files and parses them for the context manager.
parse_line takes a io and outputs it all as a list of parsed elements
Usage:
- from bootstraparse.modules.parser import parse_line
- parse_line(io) -> [element, element, element]
"""
from io import StringIO
import bootstraparse.modules.syntax as syntax
@ -12,10 +13,10 @@ import bootstraparse.modules.syntax as syntax
def parse_line(io):
"""
Takes an io string and returns the parsed output.
Parameters:
io (StringIO): The io string to parse.
Returns:
list[syntax.SemanticType]: The parsed output.
:param io: The io string to parse.
:type io: StringIO
:return: The parsed output.
:rtype: list[syntax.SemanticType]
"""
output = []
for line in io.readlines():

View File

@ -1,27 +1,33 @@
# Module for resolving relative path problems
# PathResolver takes a relative path in the module and outputs the absolute path,
# BoostraPath can deal with relative paths in the installation folder.
# Usage:
# from bootstraparse.modules.pathresolver import PathResolver, b_path
# b_path("../relative/path/to/file") # returns absolute path to file from the bootstraparse folder
# path= PathResolver("../relative/path/to/file") # Make a PathResolver object pointing at the first path given
"""
Module for resolving relative path problems
PathResolver takes a relative path in the module and outputs the absolute path,
BoostraPath can deal with relative paths in the installation folder.
Usage:
- from bootstraparse.modules.pathresolver import PathResolver, b_path
- b_path("../relative/path/to/file") # returns absolute path to file from the bootstraparse folder
- path= PathResolver("../relative/path/to/file") # Make a PathResolver object pointing at the first path given
"""
import os
class PathResolver:
"""
Initialised with a relative path and returns the translated absolute path
:param base_path: Relative path to be resolved
"""
def __init__(self, base_path):
"""
Initialises the PathResolver with the base path
:param base_path: Base path
:type base_path: str
"""
self.base_path = os.path.normpath(os.path.join(os.getcwd(), os.path.dirname(base_path)))
def __call__(self, relative_path='.'):
"""
Returns the absolute path of the relative path
:param relative_path: Relative path to be resolved
:type relative_path: str
:return: Absolute path
"""
return self.give_absolute(relative_path)
@ -30,6 +36,7 @@ class PathResolver:
"""
Returns the absolute path of the relative path
:param relative_path: Relative path to be resolved
:type relative_path: str
:return: Absolute path
"""
return os.path.normpath(os.path.join(self.base_path, relative_path))

View File

@ -1,21 +1,21 @@
# Module for pre-parsing user files in preparation for the parser
# Usage:
# from bootstraparse.modules.preparser import preparser
# pp = preparser(file, enviroment)
# pp.do_import() # imports all the modules and adds them to the file, do the same for all the files that are to be imported # noqa
# pp.do_replacements() # replaces all images and shortcuts in the file
# pp.readlines() # returns the lines of ORIGINAL file
# pp.get_all_lines() # returns the lines of the file after replacements and imports
"""Module for pre-parsing user files in preparation for the parser
Usage:
- from bootstraparse.modules.preparser import preparser
- pp = preparser(file, enviroment)
- pp.do_import() # imports all the modules and adds them to the file, do the same for all the files that are to be imported - # noqa
- pp.do_replacements() # replaces all images and shortcuts in the file
- pp.readlines() # returns the lines of ORIGINAL file
- pp.get_all_lines() # returns the lines of the file after replacements and imports
"""
import os
import regex # future: remove regex
from io import StringIO
from bootstraparse.modules import pathresolver as pr
from bootstraparse.modules import environment
from bootstraparse.modules import syntax
from bootstraparse.modules import error_mngr # noqa # pylint: disable=unused-import
from bootstraparse.modules import error_mngr
from bootstraparse.modules import export
import rich
@ -37,6 +37,10 @@ class PreParser:
:param _env: the environment object
:param list_of_paths: the list of files that have been imported in this branch of the import tree
:param dict_of_imports: Dictionary of all imports made to avoid duplicate file opening / pre-parsing
:type file_path: str
:type _env: environment.Environment
:type list_of_paths: list[str]
:type dict_of_imports: dict[str, PreParser]
"""
if list_of_paths is None:
list_of_paths = []
@ -75,7 +79,6 @@ class PreParser:
def make_temporary_files(self):
"""
Creates temporary files for the import list and the replacement.
:return: None
"""
self.file_with_all_imports = StringIO()
self.file_with_all_replacements = StringIO()
@ -86,13 +89,14 @@ class PreParser:
"""
Make new temporary files, if the old one are not referenced,
the garbage collector will delete them.
:return: None
"""
self.make_temporary_files()
def do_imports(self):
"""
Execute all actions needed to do the imports and setup for the next step
:return: The file descriptor of the file with all imports
:rtype: StringIO
"""
self.parse_import_list()
self.make_import_list()
@ -103,6 +107,8 @@ class PreParser:
def do_replacements(self):
"""
Execute all actions needed to do the replacements.
:return: The file descriptor of the file with all replacements
:rtype: StringIO
"""
self.do_imports()
self.parse_shortcuts_and_images()
@ -112,7 +118,8 @@ class PreParser:
def readlines(self):
"""
Reads the original file and returns a list of lines.
:return: A list of lines
:return: a list of lines
:rtype: list[str]
"""
with open(self.relative_path_resolver(self.name), 'r') as f:
return f.readlines()
@ -121,6 +128,7 @@ class PreParser:
"""
Get the lines from the file on the step you are in
:return: A list of lines
:rtype: list[str]
"""
if self.current_origin_for_read is None:
return self.readlines()
@ -133,6 +141,7 @@ class PreParser:
Makes sure that the files are not already imported through a previous import statement.
Recursively build a list of PreParser object for each file to be imported.
:return: the dictionary of all files to be imported (key: file name, value: PreParser object)
:rtype: dict[str, PreParser]
"""
import_list = self.parse_import_list()
if self.is_global_dict_of_imports_initialized:
@ -144,8 +153,7 @@ class PreParser:
RecursionError(f"Error: {e} was imported earlier in {self.list_of_paths}"), level='CRITICAL'
)
if e in self.global_dict_of_imports:
pp = self.global_dict_of_imports[e]
self.local_dict_of_imports[e] = pp
self.local_dict_of_imports[e] = self.global_dict_of_imports[e]
else:
try:
pp = PreParser(e, self._env, self.list_of_paths.copy(), self.global_dict_of_imports)
@ -162,7 +170,8 @@ class PreParser:
def parse_import_list(self):
"""
Parses the import list of the file.
:return: a list of files to be imported
:return: a list of files to be imported, and the line number of the import statement
:rtype: list[(str, int)]
"""
if self.saved_import_list:
return self.saved_import_list
@ -170,10 +179,10 @@ class PreParser:
line_count = 0
for line in self.readlines():
results = regex.match(_rgx_import_file, line)
results = _rgx_import_file.searchString(line)
if results:
for e in results.captures('file_name'):
import_list += [(e, line_count)]
for e in results[0]:
import_list += [(e.rstrip(), line_count)]
line_count += 1
# converts relative paths to absolute and returns a table
self.saved_import_list = [(self.relative_path_resolver(p), l) for p, l in import_list]
@ -183,6 +192,7 @@ class PreParser:
"""
Return the file object with all file imports done
:return: a filelike object with all file imports done
:rtype: StringIO
"""
self.make_import_list()
@ -202,9 +212,9 @@ class PreParser:
import_list = self.parse_import_list()
source_lines = self.readlines()
for import_path, import_line in import_list:
source_lines[import_line] = ""
temp_file.writelines(source_lines[source_line_count:import_line])
source_line_count = import_line
source_lines[import_line] = "" # remove the line where the import was
temp_file.writelines(source_lines[source_line_count:import_line]) # copy origin to destination
source_line_count = import_line # update origin for next import
import_file = self.global_dict_of_imports[import_path].export_with_imports()
temp_file.writelines(import_file.readlines())
temp_file.writelines(source_lines[source_line_count:])
@ -217,7 +227,8 @@ class PreParser:
"""
Parses through the output files from export_with_imports
and replaces shortcuts and images calls with appropriate html
:return: a table of occurrences and their lines
:return: The file descriptor of the file with all replacements
:rtype: StringIO
"""
temp_file = self.file_with_all_imports
temp_text = ''
@ -238,7 +249,10 @@ class PreParser:
def get_element_from_config(self, *list_keys):
"""
Fetches an element from the config
Fetches an element from the config (a nested dictionary) going through the list of keys
Raises an error if the element is not found
:param list_keys: the list of keys to go through the config
:type list_keys: str
"""
sub_dict = self._env.config.loaded_conf
validated_elements = []
@ -258,6 +272,12 @@ class PreParser:
def make_replacements(message, *var_list, **var_dict):
"""
Tries to format the message or give an error message and return the original message
:param message: the message to format
:type message: str
:param var_list: the list of variables to replace in the message
:type var_list: str
:keyword var_dict: the dictionary of variables to replace in the message
:type var_dict: dict[str, str]
"""
try:
return message.format(*var_list, **var_dict)
@ -271,10 +291,13 @@ class PreParser:
def get_alias_from_config(self, shortcut, optionals):
"""
Fetches shortcut paths from aliases.yaml
:return: the html to insert as a string
Returns the alias from the config and makes the replacements with the options provided
:param shortcut: id of the shortcut to fetch
:type shortcut: str
:param optionals: optional parameters along with alias
:type optionals: syntax.Optional
:return: the html to insert as a string
:rtype: str
"""
output = self.get_element_from_config('aliases', 'shortcuts', shortcut)
_, _, var_list, var_dict = syntax.split_optionals(optionals)
@ -283,10 +306,12 @@ class PreParser:
def get_image_from_config(self, shortcut, optionals):
"""
Fetches image paths from aliases.yaml
:return: html to insert as a string
:param shortcut: the id of the picture to fetch
Returns the image from the config and makes the replacements with the options provided
:param shortcut: id of the image to fetch
:type shortcut: str
:param optionals: optional parameters along with image
:type optionals: syntax.Optional
:return: the html to insert as a string
"""
shortcut_s = self.get_element_from_config('aliases', 'images', shortcut)
@ -299,7 +324,6 @@ class PreParser:
def __repr__(self):
"""
Returns a string representation of the PreParser object.
:return: a string representation of the PreParser object
"""
return "PreParser[{}](path={}, name={}, base_path={}, " \
"relative_path_resolver={}, list_of_paths={}, " \
@ -311,7 +335,6 @@ class PreParser:
def __str__(self):
"""
Returns a string representation of the PreParser object.
:return: a string representation of the PreParser object
"""
return self.__repr__()
@ -334,6 +357,7 @@ class PreParser:
"""
Checks if the PreParser object is not equal to another PreParser object.
:param other: the other PreParser object
:type other: PreParser
:return: True if the PreParser objects are not equal, False otherwise
"""
return not self.__eq__(other)
@ -341,6 +365,14 @@ class PreParser:
def rich_tree(self, prefix="", suffix="", force=True, strip_prefix=""):
"""
Returns a rich representation of the PreParser object.
:param prefix: the prefix to add to the representation
:type prefix: str
:param suffix: the suffix to add to the representation
:type suffix: str
:param force: if True, the representation is always computed, even if it has already been done
:type force: bool
:param strip_prefix: the prefix to remove from the path
:type strip_prefix: str
:return: a rich representation of the PreParser object
"""
unparsed = False

View File

@ -1,10 +1,36 @@
# Module for file and directories repartition
"""
Module for file and directories repartition
Usage:
- from bootstraparse.modules import SiteCrawler
- crawler = SiteCrawler(origin_path, dest_path, _env)
- crawler.set_all_preparsers()
- crawler.copy_unparsable_files()
- for element, destination in crawler:
- - save(preparse_parse(element), destination, _env)
"""
import os
import shutil
from bootstraparse.modules import pathresolver, preparser, error_mngr, environment, export
class SiteCrawler:
"""
A sitecrawler is a generator that yields a tuple of the form (PreParser, file)
Its goal is to create a list of preparsers and files to be parsed based on
the files and directories in the initial path.
This generator is to be used in a for loop to parse all the files and produce
the final website.
"""
def __init__(self, path, destination, _env):
"""
:param path: The path to the directory to be crawled
:param destination: The path to the directory where the website will be created
:param _env: The environment object
:type path: str
:type destination: str
:type _env: environment.Environment
"""
if not os.path.exists(path):
error_mngr.log_exception(
FileNotFoundError(f'The specified path "{path}" could not be found.'),
@ -20,6 +46,7 @@ class SiteCrawler:
self.force_rewrite = self._env.config["parser_config"]["export"]["force_rewrite"]
self.directories = []
self.files = []
self.files_to_copy = []
self.preparsers = []
self.global_dict_of_imports = {}
@ -32,26 +59,48 @@ class SiteCrawler:
self.create_all_paths()
def get_all_paths(self):
self.files, self.directories = self.list_recursively(self.initial_path, self.initial_path)
"""
This method is used to get all the paths to be crawled.
Its goal is to get all the paths to be crawled and to store them in
the self.directories and self.files, self.files_to_copy variables.
"""
self.files, self.files_to_copy, self.directories = self.list_recursively(self.initial_path, self.initial_path)
def list_recursively(self, path, root):
"""
Recursive method to get all the paths to be crawled.
:param path: The current path to be crawled
:param root: The root path (initial path)
:type path: str
:type root: str
:return: A tuple of the form (files, directories)
:rtype: (list[(str, str)], list[(str, str)], list[(str, str)])
"""
files = []
directories = []
files_to_copy = []
for element in os.listdir(path):
element_fpath = os.path.join(path, element)
element_rpath = os.path.relpath(path, root)
if os.path.isdir(element_fpath):
if element not in self.forbidden_folders:
directories.append((element_rpath, element))
f, d = self.list_recursively(element_fpath, root)
f, fc, d = self.list_recursively(element_fpath, root)
files += f
directories += d
files_to_copy += fc
else:
if os.path.splitext(element)[1] in self.authorised_extensions and element[0] != "_":
if element[0] != "_":
if os.path.splitext(element)[1] in self.authorised_extensions:
files.append((element_rpath, element))
return files, directories
else:
files_to_copy.append((element_rpath, element))
return files, files_to_copy, directories
def create_all_paths(self):
"""
This method is used to create all the directories in the destination path.
"""
if not os.path.exists(self.destination_path):
os.mkdir(self.destination_path)
for root, di in self.directories:
@ -60,6 +109,12 @@ class SiteCrawler:
os.mkdir(temp_path)
def set_all_preparsers(self):
"""
This method is used to set all the preparsers and initialize them.
The preparsers are stored in the self.preparsers variable.
:return: self.preparsers
:rtype: list[preparser.PreParser]
"""
for root, file in self.files:
preparser_path = os.path.join(self.initial_path, root, file)
pp = preparser.PreParser(preparser_path, self._env, dict_of_imports=self.global_dict_of_imports)
@ -69,7 +124,21 @@ class SiteCrawler:
return self.preparsers
def copy_unparsable_files(self):
"""
This method is used to copy all the files that could not be parsed.
"""
if self._env.config["parser_config"]["export"]["copy_unparsable_files"].lower() == "copy":
for root, file in self.files_to_copy:
shutil.copy(os.path.join(self.initial_path, root, file), os.path.join(self.destination_path, root, file))
def create_file(self, path):
"""
This method is used to create a file.
:raises FileExistsError: If the file already exists and the force_rewrite option is False
:param path: The path to the file to be created
:type path: str
"""
if os.path.exists(path):
if not self.force_rewrite:
error_mngr.log_exception(
@ -81,6 +150,11 @@ class SiteCrawler:
return path
def __iter__(self):
"""
This method is used to iterate over the preparsers and files.
:yield: A tuple of the form (PreParser, file)
:ytype: (preparser.PreParser, str)
"""
for pre in self.preparsers:
yield pre

View File

@ -1,4 +1,9 @@
# Module sequencing the successive actions necessary for website building
"""
Module sequencing the successive actions necessary for website building
Usage:
- create_website(origin, destination)
"""
import os
from bootstraparse.modules import pathresolver, sitecrawler, environment, config, export, parser, context_mngr
@ -8,10 +13,16 @@ def create_website(origin, destination):
"""
First function called by bparse.py,
calls all other modules in the right order.
:param origin: The path of the website to be built.
:param destination: The destination path of the built website.
:type origin: str
:type destination: str
:return: 0 if everything went well, 1 otherwise.
"""
env = create_environment(origin, destination)
crwlr = create_crawler(origin, destination, env)
crwlr.set_all_preparsers()
crwlr.copy_unparsable_files()
for element, destination in crwlr:
save(preparse_parse(element), destination, env)
@ -22,6 +33,12 @@ def create_environment(origin, destination):
"""
Returns parserEnvironment as an object containing
everything needed for app execution.
:param origin: The path of the website to be built.
:param destination: The destination path of the built website.
:type origin: str
:type destination: str
:return: Environment object.
:rtype: environment.Environment
"""
env = environment.Environment()
env.config = config.ConfigLoader(pathresolver.b_path("configs"))
@ -42,11 +59,26 @@ def create_environment(origin, destination):
def create_crawler(origin, destination, _env):
"""
Returns crawler as an object for navigation in the user files.
:param origin: The path of the website to be built.
:param destination: The destination path of the built website.
:param _env: The environment object.
:type origin: str
:type destination: str
:type _env: environment.Environment
:return: Crawler object.
:rtype: sitecrawler.SiteCrawler
"""
return sitecrawler.SiteCrawler(origin, destination, _env)
def preparse_parse(preparser):
"""
Returns a list of containers from a preparser.
:param preparser: The preparser object.
:type preparser: parser.Preparser
:return: List of containers.
:rtype: list
"""
io = preparser.do_replacements()
parsed_list = parser.parse_line(io)
output = context_mngr.ContextManager(parsed_list, name=preparser.name)()
@ -54,6 +86,15 @@ def preparse_parse(preparser):
def save(list_of_containers, destination, env):
"""
Saves the list of containers in the destination path.
:param list_of_containers: The list of containers to be saved.
:param destination: The destination path.
:param env: The environment object.
:type list_of_containers: list
:type destination: str
:type env: environment.Environment
"""
with open(destination, "w") as output_file:
output_file.write(export.ContextConverter(list_of_containers, env.export_mngr, destination).process_pile().read())

View File

@ -1,3 +1,4 @@
"""
# Dedicated module for the syntax of all the parsing
# All tokens inherit a SemanticType among SemanticType, ExplicitSemanticType and EmptySemanticType
# All tokens have a label, and __eq__ and __ne__ methods.
@ -8,6 +9,7 @@
# line_to_replace.parse_line('string') # returns a List of tokens parsed for replacements
# imports.parse_line('string', True) # returns a List of tokens parsed for imports
# any_token.create_diagram("filename") # Debugging
"""
import os
from itertools import zip_longest
@ -16,7 +18,6 @@ from collections import namedtuple
from bootstraparse.modules.error_mngr import MismatchedContainerError
from bootstraparse.modules import context_mngr as cm
import pyparsing as pp
import regex # future: remove regex
pps = pp.Suppress
@ -29,6 +30,11 @@ class SemanticType:
label = None
def __init__(self, content):
"""
Initialises the token.
:param content: The content of the token.
:type content: list[pp.ParseResults]
"""
self.content = content
self.line_number = "Undefined"
self.file_name = "Undefined"
@ -37,6 +43,8 @@ class SemanticType:
def to_markup(self):
"""
Function used for testing and readability purposes. Replaces matched markup with html-like tags.
:return: The markup of the token.
:rtype: str
"""
if self.content:
return_list = []
@ -51,6 +59,8 @@ class SemanticType:
def to_original(self):
"""
Return the original string of the token.
:return: The original string of the token.
:rtype: str
"""
return " ".join(self.content)
@ -79,36 +89,40 @@ class SemanticType:
return not self.__eq__(other)
def counterpart(self): # noqa # FUTURE: Check for static counterpart implementation
"""
Function called by the context manager to know how to deal with the encapsulation of the token.
Will return the counterpart of the token (the previous token to look for).
:return: The counterpart of the token.
:rtype: str | None
"""
return None
def to_container(self, filter_func=None):
"""
Function called by the context manager to know how to deal with the encapsulation of the token.
Parameters
----------
filter_func: (function | None)
A function that takes a token and returns a boolean. If the token is not valid, it should return False.
Returns
-------
SemanticType
Self if containable, the expected replacement token if it can, raises an error otherwise.
Raises
------
CannotBeContainedError
Raises error if the token we were trying to encapsulate cannot be (probably mismatched).
:param filter_func: A function that will be used to filter the content of the token.
:type filter_func: (callable| None)
:return: Self if containable, the expected replacement token if it can, raises an error otherwise.
:rtype: SemanticType | context_mngr.BaseContainer
:raises CannotBeContainedError: Raises error if the token we were trying to encapsulate cannot be (probably mismatched).
"""
raise MismatchedContainerError(self)
class TokensToMatch:
"""Class used for verification in the context manager.
Classes inheriting this should be added to matched_elements."""
class TokensToMatch(SemanticType):
"""
Class used for verification in the context manager.
Classes inheriting this should be added to matched_elements.
"""
pass
class AddFirstElementToLabel(SemanticType):
"""
Class used for verification in the context manager.
Adds the first element of the content to the label (for matching purposes).
"""
def __init__(self, content):
super().__init__(content)
self.label += ":" + content[0]
@ -247,12 +261,12 @@ class DisplayToken(FinalSemanticType):
class StructuralElementStartToken(AddFirstElementToLabel, OpenedSemanticType, TokensToMatch):
"""<<div|article|section|aside"""
"""<<div|article|section|aside|header|body|nav"""
label = 'se:start'
class StructuralElementEndToken(AddFirstElementToLabel, ClosedSemanticType):
"""div|article|section|aside>>"""
"""div|article|section|aside|header|body|nav>>"""
label = "se:end"
def counterpart(self):
@ -330,7 +344,9 @@ def of_type(token_class):
"""
Function creating a custom function for generating the given Token type.
:param token_class: SemanticType class
:type token_class: type
:return: returns a function creating an instance of the given class
:rtype: function
"""
def _of_type(_, __, content):
if len(content) == 0: # Drop Empty token
@ -344,7 +360,9 @@ def reparse(parse_element):
"""
Creates a function which reparses given match with specified parsing element.
:param parse_element: pp object defining desired match
:type parse_element: pp.ParserElement
:return: returns a function parsing given match with specified parsing element
:rtype: function
"""
def _reparse(__, _, tokens):
return parse_element.parseString(tokens[0]) # Future: document this behaviour more in-depth
@ -392,6 +410,7 @@ def readable_markup(list_of_tokens):
"""
Function used for testing and readability purposes. Replaces matched markup with html-like tags.
:param list_of_tokens: list of matched markup tokens in analysed text.
:type list_of_tokens: list
:return: returns readable string.
:rtype: string
"""
@ -408,8 +427,7 @@ def readable_markup(list_of_tokens):
# Pre-parser expressions
rgx_import_file = regex.compile(r'::( ?\< ?(?P<file_name>[\w\-._/]+) ?\>[ \s]*)+')
rgx_import_file = pps("::") + pp.OneOrMore(pps("<") + pp.SkipTo(">").set_name("file_name")("file_name") + pps(">"))
# Base elements
quotes = pp.Word(r""""'""")
@ -446,7 +464,10 @@ structural_elements = (
pp.CaselessLiteral('div') |
pp.CaselessLiteral('article') |
pp.CaselessLiteral('aside') |
pp.CaselessLiteral('section')
pp.CaselessLiteral('section') |
pp.CaselessLiteral('header') |
pp.CaselessLiteral('body') |
pp.CaselessLiteral('nav')
)('structural_element')
header_element = pp.Word('#')
display_element = pp.Word('!')
@ -501,22 +522,20 @@ multi_line = se | table | quotation
one_header = (
header_element +
pp.SkipTo(pp.match_previous_literal(header_element)) +
pps(pp.match_previous_literal(header_element)) +
pp.Opt(optional)
).add_parse_action(of_type(HeaderToken))
pps(pp.match_previous_literal(header_element))
).add_parse_action(of_type(HeaderToken)) + pp.Opt(optional)
one_display = (
display_element +
pp.SkipTo(pp.match_previous_literal(display_element)) +
pps(pp.match_previous_literal(display_element)) +
pp.Opt(optional)
).add_parse_action(of_type(DisplayToken))
pps(pp.match_previous_literal(display_element))
).add_parse_action(of_type(DisplayToken)) + pp.Opt(optional)
one_olist = pp.line_start + (
pps(pp.Literal('#.')) + (
pps(pp.Combine(pp.line_start + pp.Literal('#.'))) + (
(pp.SkipTo(optional)('text').add_parse_action(reparse(enhanced_text)) + optional) |
enhanced_text)
).add_parse_action(of_type(EtOlistToken))
one_ulist = pp.line_start + (
pps(pp.Literal('-')) + (
pps(pp.Combine(pp.line_start + pp.Literal('-'))) + (
(pp.SkipTo(optional)('text').add_parse_action(reparse(enhanced_text)) + optional) |
enhanced_text)
).add_parse_action(of_type(EtUlistToken))
@ -538,10 +557,9 @@ alias = alias_element + pp.Opt(optional)
# Syntax elements
line_to_replace = pp.OneOrMore(
pp.SkipTo(image ^ alias)('text').add_parse_action(of_type(TextToken))
^ image.add_parse_action(of_type(ImageToken))
^ alias.add_parse_action(of_type(AliasToken))
) ^ pp.rest_of_line('text').add_parse_action(of_type(TextToken))
pp.SkipTo(image)('text').add_parse_action(of_type(TextToken)) + image.add_parse_action(of_type(ImageToken)) |
pp.SkipTo(alias)('text').add_parse_action(of_type(TextToken)) + alias.add_parse_action(of_type(AliasToken))
) | pp.rest_of_line('text').add_parse_action(of_type(TextToken))
##############################################################################

View File

@ -1 +1,3 @@
# Module for interpreting default or custom templates.
"""
Module for interpreting default or custom templates.
"""

View File

@ -1,17 +1,58 @@
###############################################################################
# List of tools for testing
###############################################################################
# Usage:
# from bootstraparse.modules.tools import __GLk, __GL, find_*_in_file
# __GLk(n) # Returns a file link n traceback objects back
# __GL() # Returns a line number at current position
# find_*_in_file(file) # Try to find (class, function or variable) in file and return its line number
"""
List of tools for testing
=========================
Usage:
- from bootstraparse.modules.tools import __GLk, __GL, find_*_in_file
- __GLk(n) # Returns a file link n traceback objects back
- __GL() # Returns a line number at current position
- find_*_in_file(file) # Try to find (class, function or variable) in file and return its line number
"""
import inspect
import os
import re
###############################################################################
# Dictionary checks
########################################
def dict_check(dic, *args):
"""
Helper function to test existence of a list of succesive keys within a nested dictionary.
:param dic: The dictionary of dictionaries to check
:param args: Any number of keys to check the dictionaries for
:type dic: dict | config.ConfigLoader
:type args: str
:return: A table of Booleans for every key checked.
"""
output = [False for _ in args]
for i in range(len(args)):
if args[i] in dic:
output[i] = True
dic = dic[args[i]]
else:
break
return output
def str_dict_check(dic, *args):
"""
Helper function to test existence within a dictionary and return a string of the values of the keys checked
and wether they exist.
'arg[0]' : True or False
:param dic: The nested dictionary to check
:param args: Any number of keys to check successively
:type dic: dict
:type args: str
:return: A string of the values of the keys checked and wether they exist.
:rtype: str
"""
out = dict_check(dic, *args)
return "\n".join([f"{args[i]}: {out[i]}" for i in range(len(out))])
###############################################################################
# Paths
########################################

View File

@ -1,8 +1,11 @@
structural_elements:
div: ["<div{optionals}>", "</div>"]
article: ["<article{optionals}>", "</article>"]
aside: ["<aside{optionals}>", "</aside>"]
section: ["<section{optionals}>", "</section>"]
se_div: ["<div{optionals}>", "</div>"]
se_article: ["<article{optionals}>", "</article>"]
se_aside: ["<aside{optionals}>", "</aside>"]
se_section: ["<section{optionals}>", "</section>"]
se_nav: ["<nav{optionals}>", "</nav>"]
se_header: ["<header{optionals}>", "</header>"]
se_body: ["<body{optionals}>", "</body>"]
header: ["<h{header_level}{optionals}>", "</h{header_level}>"]
display: ["<p class=\"display-{display_level}\"{optionals}>", "</p>"]

View File

@ -1,5 +1,7 @@
import os
from yaml.scanner import ScannerError
import bootstraparse.modules.config as config
import pytest
import tempfile
@ -10,7 +12,8 @@ configs = {
"app/configs/parser_config.yaml": '{"parser": {"name": "test_parser", "type": "test_parser"}}',
"app/configs/aliases.yaml": '{"parser": {"name": "test_parser", "type": "test_parser"}}',
"user/config/aliases.yaml": '{"aliases": {"test_alias": "test_parser"}}',
"user/config/glossary.yaml": '{"glossary": {"test_glossary": "test_parser"}}',
"app/configs/glossary.yaml": '{"glossary": {"test_glossary": "test_parser"}}',
"user/config/glossary.yaml": '{"glossary": {"test_glossary": "second_parser"}}',
"user/config/custom_template.yaml": '{"custom_template": {"test_template": "test_parser"}}',
}
bad_configs = {
@ -36,7 +39,9 @@ def make_false_configs():
def test_config_load():
usr_c = config.ConfigLoader(user_conf)
assert usr_c["aliases"] == {"aliases": {"test_alias": "test_parser"}}
assert usr_c["glossary"] == {"glossary": {"test_glossary": "test_parser"}}
assert usr_c["glossary"] == {"glossary": {"test_glossary": "second_parser"}}
assert usr_c["glossary"]["glossary"] == {"test_glossary": "second_parser"}
assert usr_c["glossary"]["glossary"]["test_glossary"] == "second_parser"
assert usr_c["aliases"]["aliases"] == {"test_alias": "test_parser"}
assert usr_c["aliases"]["aliases"]["test_alias"] == "test_parser"
with pytest.raises(KeyError):
@ -54,7 +59,6 @@ def test_add_to_config():
assert from_list["aliases"] == {"aliases": {"test_alias": "test_parser"},
"parser": {"name": "test_parser", "type": "test_parser"}}
assert from_list["parser_config"] == {"parser": {"name": "test_parser", "type": "test_parser"}}
with pytest.raises(KeyError): # Not FileNotFoundError
assert from_empty["aliases"]
from_empty.add_folder(user_conf)
@ -64,7 +68,7 @@ def test_add_to_config():
assert from_empty["aliases"] == {"aliases": {"test_alias": "test_parser"},
"parser": {"name": "test_parser", "type": "test_parser"}}
with pytest.raises(TypeError):
with pytest.raises(ScannerError):
from_empty.load_from_file(os.path.join(bad_conf, "bad_config.yaml"))

View File

@ -64,6 +64,21 @@ _token_list_with_expected_result = [
],
__GLk(1),
],
[
[
sy.StructuralElementStartToken(["header"]),
sy.TextToken(["1"]),
sy.StructuralElementEndToken(["header"]),
],
[
context_mngr.SeContainer([
sy.StructuralElementStartToken(["header"]),
context_mngr.TextContainer([sy.TextToken(["1"])]),
sy.StructuralElementEndToken(["header"])
]),
],
__GLk(1),
],
[
[
sy.EtEmToken(["*"]),

View File

@ -4,6 +4,7 @@ from collections import namedtuple
from importlib import reload
from unittest import TestCase
import pytest
import bootstraparse.modules.tools as tools
import bootstraparse.modules.error_mngr as error_mngr
from bootstraparse.modules.error_mngr import ParsingError, MismatchedContainerError
@ -83,23 +84,23 @@ def test_dict_check():
"""
Test the dict_check function
"""
assert(error_mngr.dict_check({"test": "test"}, "test") == [True])
assert(error_mngr.dict_check({"test": "test"}, "test2") == [False])
assert(error_mngr.dict_check({"test": {"test3": "test5"}}, "test", "test3") == [True, True])
assert(error_mngr.dict_check({"test": {"test3": "test5"}}, "test4", "test4") == [False, False])
assert(error_mngr.dict_check({"test": {"test3": "test5"}}, "test", "test34") == [True, False])
assert(tools.dict_check({"test": "test"}, "test") == [True])
assert(tools.dict_check({"test": "test"}, "test2") == [False])
assert(tools.dict_check({"test": {"test3": "test5"}}, "test", "test3") == [True, True])
assert(tools.dict_check({"test": {"test3": "test5"}}, "test4", "test4") == [False, False])
assert(tools.dict_check({"test": {"test3": "test5"}}, "test", "test34") == [True, False])
# @pytest.mark.xfail(strict=True, reason="This test is not implemented yet")
def test_dict_check_2():
assert(error_mngr.dict_check({"test": "test"}, "test", "test2") == [True, False])
assert(error_mngr.dict_check({"test": "test"}, "test", "test2", "test3") == [True, False, False])
assert(tools.dict_check({"test": "test"}, "test", "test2") == [True, False])
assert(tools.dict_check({"test": "test"}, "test", "test2", "test3") == [True, False, False])
def test_dict_str():
assert error_mngr.str_dict_check({"test": "test"}, "test") == "test: True"
assert error_mngr.str_dict_check({"test": "test"}, "test2") == "test2: False"
assert error_mngr.str_dict_check({"test": {"test3": "test5"}}, "test", "test3") == "test: True\ntest3: True"
assert tools.str_dict_check({"test": "test"}, "test") == "test: True"
assert tools.str_dict_check({"test": "test"}, "test2") == "test2: False"
assert tools.str_dict_check({"test": {"test3": "test5"}}, "test", "test3") == "test: True\ntest3: True"
def test_weird_exception_cases():

View File

@ -12,12 +12,14 @@ __templates = config.ConfigLoader(pathresolver.b_path("templates/"))
__XF = pytest.mark.xfail
_all_templates = {
"structural_elements": [
["div", __GLk(1)],
["article", __GLk(1)],
["aside", __GLk(1)],
["section", __GLk(1)],
["se_div", __GLk(1)],
["se_article", __GLk(1)],
["se_aside", __GLk(1)],
["se_section", __GLk(1)],
["header", __GLk(1)],
["display", __GLk(1)],
["se_body", __GLk(1)],
["se_nav", __GLk(1)],
],
"inline_elements": [
["link", __GLk(1)],
@ -86,7 +88,7 @@ def test_export_response():
("table", "t_head"),
("table", "t_row"),
("table", "t_cell"),
("structural_elements", "div"),
("structural_elements", "se_div"),
])
def test_transform(export_type, export_subtype):
"""
@ -118,7 +120,7 @@ def test_get_template_error():
def test_with_optionnals():
em = export.ExportManager(__config, __templates)
em(export.ExportRequest("structural_elements", "div", _opts)) # noqa E741
em(export.ExportRequest("structural_elements", "se_div", _opts)) # noqa E741
def test_format_optionnals():
@ -147,6 +149,7 @@ def test_bad_header():
@pytest.mark.parametrize("export_type, export_subtype, line", zipped_templates)
def test_get_templates(export_type, export_subtype, line):
print(line)
em = export.ExportManager(__config, __templates)
# assert type(em.get_templates()) is dict
print(export_type, export_subtype)

View File

@ -246,7 +246,7 @@ def test_make_import_list(filename, content, capsys):
])
def test_preparser_content(filename, content):
"""
Test the preparser content
Test the PreParser content
"""
testfile = temp_name(os.path.join(_BASE_PATH_GIVEN, filename))
assert os.path.exists(testfile)
@ -258,7 +258,7 @@ def test_preparser_content(filename, content):
def test_subfolder_parsing():
"""
Test the preparser content
Test the PreParser content
"""
testfile = temp_name(os.path.join(_BASE_PATH_GIVEN, "superimports.bpr"))
assert os.path.exists(testfile)

View File

@ -19,12 +19,21 @@ files = [
("base/config/test6.bpr", "", None),
("base/config/test7.yml", "", None),
("base/templates/test8.yml", "", None),
("base/unparsable.php", "", ""),
]
def make_new_file(path, content="", mode="w+"):
"""
Make a new file
Make a new file and all the directories in the path
:param path: the path to the file
:type path: str
:param content: the content of the file
:type content: str
:param mode: the mode of the file
:type mode: str
:return: the path to the file
:rtype: str
"""
name = os.path.join(_TEMP_DIRECTORY.name, path)
os.makedirs(os.path.dirname(name), exist_ok=True)
@ -88,6 +97,7 @@ def test_result_crawler(list_files, env):
"""
crw = sitecrawler.SiteCrawler(_BASE, _DEST, env)
crw.set_all_preparsers()
crw.copy_unparsable_files()
for pp, dest in crw:
assert os.path.exists(dest)
assert os.path.isfile(dest)

View File

@ -4,7 +4,7 @@ import pyparsing
import pytest
import bootstraparse.modules.syntax as sy
from bootstraparse.modules.tools import __GL, __module_path, find_variables_in_file
from bootstraparse.modules.tools import __GL, __GLk, __module_path, find_variables_in_file
ptp = pytest.param
__XF = pytest.mark.xfail
@ -111,33 +111,33 @@ dict_advanced_syntax_input_and_expected_output = {
# Enhanced text
"et_em": [
# Single em element, should only match the first "*"
("*", (sy.EtEmToken(["*"]),), __GL()),
("*", (sy.EtEmToken(["*"]),), __GLk()),
],
"et_strong": [
# Single strong element, should only match the first "**"
("**", (sy.EtStrongToken(["**"]),), __GL()),
("**", (sy.EtStrongToken(["**"]),), __GLk()),
],
"et_underline": [
# Single underline element, should only match the first "__"
("__", (sy.EtUnderlineToken(["__"]),), __GL()),
("__", (sy.EtUnderlineToken(["__"]),), __GLk()),
],
"et_strikethrough": [
# Single strikethrough element, should only match the first "~~"
("~~", (sy.EtStrikethroughToken(["~~"]),), __GL()),
("~~", (sy.EtStrikethroughToken(["~~"]),), __GLk()),
],
"et_custom_span": [
# Single custom span element, should only match the first "(#number)"
("(#12345)", (sy.EtCustomSpanToken(["12345"]),), __GL()),
("(#12345)", (sy.EtCustomSpanToken(["12345"]),), __GLk()),
],
"il_link": [
# Single link element, should only match the first "[link_name]('link')"
("[text_link]('text://www.website.com/link.html')",
[sy.HyperlinkToken(["[text_link]('text://www.website.com/link.html')"])], __GL())
[sy.HyperlinkToken(["[text_link]('text://www.website.com/link.html')"])], __GLk())
],
"assignation": [
# Single assignation element, should only match the first "a=1"
("a=1", [sy.BeAssignToken([["a", 1]]), ], __GL()),
("a='str'", [sy.BeAssignToken([["a", 'str']]), ], __GL()),
("a=1", [sy.BeAssignToken([["a", 1]]), ], __GLk()),
("a='str'", [sy.BeAssignToken([["a", 'str']]), ], __GLk()),
],
"var": [
# List of variables or assignations, should match "[a=12, b=13, 'c']"
@ -145,13 +145,13 @@ dict_advanced_syntax_input_and_expected_output = {
sy.BeAssignToken([["a_12", 12]]),
sy.BeAssignToken([["b", 13]]),
sy.BeValueToken(["c"]),
], __GL()),
], __GLk()),
("[a=12, b=13, 'c', d=14]", [
sy.BeAssignToken([["a", 12]]),
sy.BeAssignToken([["b", 13]]),
sy.BeValueToken(["c"]),
sy.BeAssignToken([["d", 14]]),
], __GL()),
], __GLk()),
],
"optional": [
# Optional element, should match any optional element
@ -163,7 +163,7 @@ dict_advanced_syntax_input_and_expected_output = {
sy.BeValueToken(["c"]),
]),
]),
], __GL()),
], __GLk()),
("[a=12, b=13, 'c', d=14]", [
sy.OptionalToken([
sy.OptionalVarToken([
@ -173,7 +173,7 @@ dict_advanced_syntax_input_and_expected_output = {
sy.BeAssignToken([["d", 14]]),
]),
]),
], __GL()),
], __GLk()),
("[a=12]{aze,/.}", (
sy.OptionalToken([
sy.OptionalVarToken([
@ -183,7 +183,7 @@ dict_advanced_syntax_input_and_expected_output = {
"aze,/.",
]),
]),
), __GL()),
), __GLk()),
("{aze,/.__}[a=12]", (
sy.OptionalToken([
sy.OptionalInsertToken([
@ -193,21 +193,21 @@ dict_advanced_syntax_input_and_expected_output = {
sy.BeAssignToken([["a", 12]]),
]),
]),
), __GL()),
), __GLk()),
("{aze!=}", (
sy.OptionalToken([
sy.OptionalInsertToken([
"aze!=",
]),
]),
), __GL()),
), __GLk()),
("{{aze!=}}", (
sy.OptionalToken([
sy.OptionalClassToken([
"aze!=",
]),
]),
), __GL()),
), __GLk()),
("{{aze!=}}{azett!=}", (
sy.OptionalToken([
sy.OptionalClassToken([
@ -217,7 +217,7 @@ dict_advanced_syntax_input_and_expected_output = {
"azett!=",
]),
]),
), __GL()),
), __GLk()),
("{{aze!=}}{azett!=}[a_32=12]", (
sy.OptionalToken([
sy.OptionalClassToken([
@ -230,7 +230,7 @@ dict_advanced_syntax_input_and_expected_output = {
sy.BeAssignToken([["a_32", 12]]),
]),
]),
), __GL()),
), __GLk()),
("[_a=12]{aze,/.}{{azer!=}}", (
sy.OptionalToken([
sy.OptionalVarToken([
@ -243,36 +243,36 @@ dict_advanced_syntax_input_and_expected_output = {
"azer!=",
]),
]),
), __GL()),
), __GLk()),
],
# Enhanced Text
"enhanced_text": [
# Matches a line of text, with or without inline elements
("*test*", (sy.EtEmToken(["*"]), sy.TextToken(["test"]), sy.EtEmToken(["*"])), __GL()),
("**test**", (sy.EtStrongToken(["**"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"])), __GL()),
("__test__", (sy.EtUnderlineToken(["__"]), sy.TextToken(["test"]), sy.EtUnderlineToken(["__"])), __GL()),
("~~test~~", (sy.EtStrikethroughToken(["~~"]), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"])), __GL()),
("(#12345)", (sy.EtCustomSpanToken(["12345"]),), __GL()),
("*test*", (sy.EtEmToken(["*"]), sy.TextToken(["test"]), sy.EtEmToken(["*"])), __GLk()),
("**test**", (sy.EtStrongToken(["**"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"])), __GLk()),
("__test__", (sy.EtUnderlineToken(["__"]), sy.TextToken(["test"]), sy.EtUnderlineToken(["__"])), __GLk()),
("~~test~~", (sy.EtStrikethroughToken(["~~"]), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"])), __GLk()),
("(#12345)", (sy.EtCustomSpanToken(["12345"]),), __GLk()),
("*test*test*", (sy.EtEmToken(["*"]), sy.TextToken(["test"]), sy.EtEmToken(["*"]), sy.TextToken(["test"]),
sy.EtEmToken(["*"])), __GL()),
sy.EtEmToken(["*"])), __GLk()),
("**test**test**",
(sy.EtStrongToken(["**"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]), sy.TextToken(["test"]),
sy.EtStrongToken(["**"]),), __GL()),
sy.EtStrongToken(["**"]),), __GLk()),
("__test__test__", (sy.EtUnderlineToken(["__"]), sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]),
sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]),), __GL()),
sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]),), __GLk()),
("~~test~~test~~", (sy.EtStrikethroughToken(["~~"]), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]),
sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]),), __GL()),
sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]),), __GLk()),
("(#12345)test(#12345)", (sy.EtCustomSpanToken(["12345"]), sy.TextToken(["test"]),
sy.EtCustomSpanToken(["12345"]),), __GL()),
sy.EtCustomSpanToken(["12345"]),), __GLk()),
("__test(#12)test**test__test~~", (sy.EtUnderlineToken(["__"]), sy.TextToken(["test"]),
sy.EtCustomSpanToken(["12"]), sy.TextToken(["test"]),
sy.EtStrongToken(["**"]),
sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]),
sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]),), __GL()),
sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]),), __GLk()),
("Test text with a link [link_name]('link')", (
sy.TextToken(["Test text with a link"]),
sy.HyperlinkToken(["[link_name]('link')"]),
), __GL()),
), __GLk()),
("Test text with a link [link_name](\"link\") and a *bold* text", (
sy.TextToken(["Test text with a link"]),
sy.HyperlinkToken(['[link_name]("link")']),
@ -280,39 +280,39 @@ dict_advanced_syntax_input_and_expected_output = {
sy.TextToken(["bold"]),
sy.EtEmToken(["*"]),
sy.TextToken(["text"]),
), __GL()),
), __GLk()),
("Reverse order (#123) Span __ Underline ~~ Strikethrough", (
sy.TextToken(["Reverse order"]),
sy.EtCustomSpanToken(["123"]), sy.TextToken(["Span"]),
sy.EtUnderlineToken(["__"]), sy.TextToken(["Underline"]),
sy.EtStrikethroughToken(["~~"]), sy.TextToken(["Strikethrough"]),
), __GL()),
), __GLk()),
],
"html_insert": [
# Match an HTML insert
("{azer='\"}", (
"azer='\"",
), __GL()),
), __GLk()),
],
"class_insert": [
# Match a class insert
("{{azer='\"}}", (
"azer='\"",
), __GL()),
), __GLk()),
],
# Structural Elements
"se_start": [
# Matches a start of a structural element
("<<div", (sy.StructuralElementStartToken(["div"]),), __GL()),
("<<aside", (sy.StructuralElementStartToken(["aside"]),), __GL()),
("<<article", (sy.StructuralElementStartToken(["article"]),), __GL()),
("<<section", (sy.StructuralElementStartToken(["section"]),), __GL()),
("<<div", (sy.StructuralElementStartToken(["div"]),), __GLk()),
("<<aside", (sy.StructuralElementStartToken(["aside"]),), __GLk()),
("<<article", (sy.StructuralElementStartToken(["article"]),), __GLk()),
("<<section", (sy.StructuralElementStartToken(["section"]),), __GLk()),
],
"se_end": [
("div>>", (sy.StructuralElementEndToken(["div"]),), __GL()),
("aside>>", (sy.StructuralElementEndToken(["aside"]),), __GL()),
("article>>", (sy.StructuralElementEndToken(["article"]),), __GL()),
("section>>", (sy.StructuralElementEndToken(["section"]),), __GL()),
("div>>", (sy.StructuralElementEndToken(["div"]),), __GLk()),
("aside>>", (sy.StructuralElementEndToken(["aside"]),), __GLk()),
("article>>", (sy.StructuralElementEndToken(["article"]),), __GLk()),
("section>>", (sy.StructuralElementEndToken(["section"]),), __GLk()),
],
"se": [
# Matches a div element, must be at the beginning of the line, the closing div can be with arguments
@ -325,7 +325,7 @@ dict_advanced_syntax_input_and_expected_output = {
]),
sy.OptionalInsertToken(["var='test', number=11"]),
]),
), __GL(),),
), __GLk(),),
("div>> [class='blue', 123]", (
sy.StructuralElementEndToken(["div"]),
sy.OptionalToken([
@ -334,56 +334,56 @@ dict_advanced_syntax_input_and_expected_output = {
sy.BeValueToken([123.0]),
]),
]),
), __GL(),),
), __GLk(),),
("div>> {var='test', number=11}", (
sy.StructuralElementEndToken(["div"]),
sy.OptionalToken([
sy.OptionalInsertToken(["var='test', number=11"]),
]),
), __GL(),),
), __GLk(),),
],
# List Elements
"one_olist": [
# Matches a one-level ordered list element, must be at the beginning of the line
# Drop the "." from the match
("#. Text", (sy.EtOlistToken([sy.TextToken(["Text"])]),), __GL()),
("#. Text", (sy.EtOlistToken([sy.TextToken(["Text"])]),), __GLk()),
],
"one_ulist": [
# Matches a one-level unordered list element, must be at the beginning of the line
# Drop the "-" from the match
("- Text", (sy.EtUlistToken([sy.TextToken(["Text"])]),), __GL()),
("- Text", (sy.EtUlistToken([sy.TextToken(["Text"])]),), __GLk()),
],
# Headers
"one_header": [
# Matches a one-level header element, must be at the beginning of the line
("# Text1 #", (sy.HeaderToken(["#", "Text1 "]),), __GL()),
("## Text2 ##", (sy.HeaderToken(["##", "Text2 "]),), __GL()),
("### Text3 ###", (sy.HeaderToken(["###", "Text3 "]),), __GL()),
("#### Text4 ####", (sy.HeaderToken(["####", "Text4 "]),), __GL()),
("##### Text5 #####", (sy.HeaderToken(["#####", "Text5 "]),), __GL()),
("###### Text6 ######", (sy.HeaderToken(["######", "Text6 "]),), __GL()),
("# Text1 #", (sy.HeaderToken(["#", "Text1 "]),), __GLk()),
("## Text2 ##", (sy.HeaderToken(["##", "Text2 "]),), __GLk()),
("### Text3 ###", (sy.HeaderToken(["###", "Text3 "]),), __GLk()),
("#### Text4 ####", (sy.HeaderToken(["####", "Text4 "]),), __GLk()),
("##### Text5 #####", (sy.HeaderToken(["#####", "Text5 "]),), __GLk()),
("###### Text6 ######", (sy.HeaderToken(["######", "Text6 "]),), __GLk()),
],
# One-line display elements
"one_display": [
# Matches a one-level display element, must be at the beginning of the line
("! Display1 !", (sy.DisplayToken(["!", "Display1 "]),), __GL()),
("!! Display2 !!", (sy.DisplayToken(["!!", "Display2 "]),), __GL()),
("!!! Display3 !!!", (sy.DisplayToken(["!!!", "Display3 "]),), __GL()),
("! Display1 !", (sy.DisplayToken(["!", "Display1 "]),), __GLk()),
("!! Display2 !!", (sy.DisplayToken(["!!", "Display2 "]),), __GLk()),
("!!! Display3 !!!", (sy.DisplayToken(["!!!", "Display3 "]),), __GLk()),
],
# One-line elements
"one_line": [
# Matches a one-line element, must be at the beginning of the line
("- Text1", (sy.EtUlistToken([sy.TextToken(["Text1"])]),), __GL()),
("#. Text2", (sy.EtOlistToken([sy.TextToken(["Text2"])]),), __GL()),
("! Display3 !", (sy.DisplayToken(["!", "Display3 "]),), __GL()),
("!! Display4 !!", (sy.DisplayToken(["!!", "Display4 "]),), __GL()),
("### Text5 ###", (sy.HeaderToken(["###", "Text5 "]),), __GL()),
("#### Text6 ####", (sy.HeaderToken(["####", "Text6 "]),), __GL()),
("- Text1", (sy.EtUlistToken([sy.TextToken(["Text1"])]),), __GLk()),
("#. Text2", (sy.EtOlistToken([sy.TextToken(["Text2"])]),), __GLk()),
("! Display3 !", (sy.DisplayToken(["!", "Display3 "]),), __GLk()),
("!! Display4 !!", (sy.DisplayToken(["!!", "Display4 "]),), __GLk()),
("### Text5 ###", (sy.HeaderToken(["###", "Text5 "]),), __GLk()),
("#### Text6 ####", (sy.HeaderToken(["####", "Text6 "]),), __GLk()),
("- Text7 [class='blue', 123]{var='test', number=11}", (
sy.EtUlistToken([
sy.TextToken(["Text7"]),
@ -395,7 +395,7 @@ dict_advanced_syntax_input_and_expected_output = {
sy.OptionalInsertToken(["var='test', number=11"]),
]),
]),
), __GL()),
), __GLk()),
("#. Text8 [class='blue', 123]{var='test', number=11}", (
sy.EtOlistToken([
sy.TextToken(["Text8"]),
@ -407,11 +407,12 @@ dict_advanced_syntax_input_and_expected_output = {
sy.OptionalInsertToken(["var='test', number=11"]),
]),
]),
), __GL()),
), __GLk()),
("! Display9 ! [class='blue', 123]{var='test', number=11}", (
sy.DisplayToken([
"!",
"Display9 ",
]),
sy.OptionalToken([
sy.OptionalVarToken([
sy.BeAssignToken([["class", "blue"]]),
@ -419,12 +420,12 @@ dict_advanced_syntax_input_and_expected_output = {
]),
sy.OptionalInsertToken(["var='test', number=11"]),
]),
]),
), __GL()), # Bad optional implementation
), __GLk(1)), # Bad optional implementation
("## Text10 ## [class='blue', 123]{var='test', number=11}", (
sy.HeaderToken([
"##",
"Text10 ",
]),
sy.OptionalToken([
sy.OptionalVarToken([
sy.BeAssignToken([["class", "blue"]]),
@ -432,8 +433,7 @@ dict_advanced_syntax_input_and_expected_output = {
]),
sy.OptionalInsertToken(["var='test', number=11"]),
]),
]),
), __GL()), # Bad optional implementation
), __GLk(1)), # Bad optional implementation
],
# Tables
@ -442,27 +442,27 @@ dict_advanced_syntax_input_and_expected_output = {
("| Text1 | Text2 |", (sy.TableRowToken([
sy.TableCellToken([sy.TextToken(["Text1 "])]),
sy.TableCellToken([sy.TextToken(["Text2 "])]),
]), ), __GL()),
]), ), __GLk()),
("| Text1 | Text2 | Text3 |", (
sy.TableRowToken([
sy.TableCellToken([sy.TextToken(["Text1 "])]),
sy.TableCellToken([sy.TextToken(["Text2 "])]),
sy.TableCellToken([sy.TextToken(["Text3 "])]),
]),
), __GL()),
), __GLk()),
("| 2 Text1 | Text2 |", (
sy.TableRowToken([
sy.TableCellToken([sy.TextToken(["2 Text1 "])]),
sy.TableCellToken([sy.TextToken(["Text2 "])]),
]),
), __GL()),
), __GLk()),
("|2 Text1 | Text2 |", (
sy.TableRowToken([
'2',
sy.TableCellToken([sy.TextToken(["Text1 "])]),
sy.TableCellToken([sy.TextToken(["Text2 "])]),
]),
), __GL()),
), __GLk()),
("|2 Text1 |3 Text2 |", (
sy.TableRowToken([
'2',
@ -470,7 +470,7 @@ dict_advanced_syntax_input_and_expected_output = {
'3',
sy.TableCellToken([sy.TextToken(["Text2 "])]),
]),
), __GL()),
), __GLk()),
("|2 Text1 |3 Text2 |4 Text3 |", (
sy.TableRowToken([
'2',
@ -480,7 +480,7 @@ dict_advanced_syntax_input_and_expected_output = {
'4',
sy.TableCellToken([sy.TextToken(["Text3 "])]),
]),
), __GL()),
), __GLk()),
("|3 Text1 | Text2 | {var='test', number=11}", (
sy.TableRowToken([
'3',
@ -492,19 +492,19 @@ dict_advanced_syntax_input_and_expected_output = {
"var='test', number=11",
]),
]),
), __GL()),
), __GLk()),
],
"table_separator": [
# Matches a table element, must be at the beginning of the line
("|---|---|", (sy.TableSeparatorToken(["---", "---"]),), __GL()),
("|---|---|---|", (sy.TableSeparatorToken(["---", "---", "---"]),), __GL()),
("|:--|-:-|--:|--:|", (sy.TableSeparatorToken([":--", "-:-", "--:", "--:"]),), __GL()),
("|---|---|", (sy.TableSeparatorToken(["---", "---"]),), __GLk()),
("|---|---|---|", (sy.TableSeparatorToken(["---", "---", "---"]),), __GLk()),
("|:--|-:-|--:|--:|", (sy.TableSeparatorToken([":--", "-:-", "--:", "--:"]),), __GLk()),
],
"line": [
# Match any line parsed by the parser (can match header, list table etc...) this is the main syntax element
("# Text1 #", (sy.HeaderToken(["#", "Text1 "]),), __GL()),
("# Text1 #", (sy.HeaderToken(["#", "Text1 "]),), __GLk()),
("Text *em __underline__ still em*", (
sy.TextToken(["Text"]),
sy.EtEmToken(["*"]),
@ -514,17 +514,19 @@ dict_advanced_syntax_input_and_expected_output = {
sy.EtUnderlineToken(["__"]),
sy.TextToken(["still em"]),
sy.EtEmToken(["*"]),
), __GL(),),
), __GLk(),),
("|2 Text1 | Text2 |", (
sy.TableRowToken([
"2",
sy.TableCellToken([sy.TextToken(["Text1 "])]),
sy.TableCellToken([sy.TextToken(["Text2 "])]),
]),
), __GL()),
("|---|---|", (sy.TableSeparatorToken(["---", "---"]),), __GL()),
("- Text", (sy.EtUlistToken([sy.TextToken(["Text"])]),), __GL()),
("div>>", (sy.StructuralElementEndToken(["div"]),), __GL()),
), __GLk()),
("|---|---|", (sy.TableSeparatorToken(["---", "---"]),), __GLk()),
("- Text", (sy.EtUlistToken([sy.TextToken(["Text"])]),), __GLk()),
(" - Non ulist", (sy.TextToken(["- Non ulist"]),), __GLk()),
("div>>", (sy.StructuralElementEndToken(["div"]),), __GLk()),
("body>>", (sy.StructuralElementEndToken(["body"]),), __GLk()),
("#. Text with *em* and __underline__ and {arguments}{{super-arguments}}[a=1, c=2,'ui']", (
sy.EtOlistToken([
sy.TextToken(["Text with"]),
@ -550,11 +552,11 @@ dict_advanced_syntax_input_and_expected_output = {
]),
]),
]),
), __GL(),),
("```Code```", (sy.CodeToken(["```"]), sy.TextToken(["Code"]), sy.CodeToken(["```"])), __GL()),
), __GLk(),),
("```Code```", (sy.CodeToken(["```"]), sy.TextToken(["Code"]), sy.CodeToken(["```"])), __GLk()),
],
"blockquote": [
("> Text", (sy.BlockQuoteToken([sy.TextToken(["Text"])]),), __GL()),
("> Text", (sy.BlockQuoteToken([sy.TextToken(["Text"])]),), __GLk()),
("> Text with *em* and __underline__", (
sy.BlockQuoteToken([
sy.TextToken(["Text with"]),
@ -566,15 +568,15 @@ dict_advanced_syntax_input_and_expected_output = {
sy.TextToken(["underline"]),
sy.EtUnderlineToken(["__"]),
]),
), __GL()),
), __GLk()),
],
"blockquote_author": [
("> -- Author Name", (
sy.BlockQuoteAuthorToken(["Author Name"]),
), __GL()),
), __GLk()),
],
"quotation": [
("> Text", (sy.BlockQuoteToken([sy.TextToken(["Text"])]),), __GL()),
("> Text", (sy.BlockQuoteToken([sy.TextToken(["Text"])]),), __GLk()),
("> Text with *em* and __underline__", (
sy.BlockQuoteToken([
sy.TextToken(["Text with"]),
@ -586,14 +588,14 @@ dict_advanced_syntax_input_and_expected_output = {
sy.TextToken(["underline"]),
sy.EtUnderlineToken(["__"]),
]),
), __GL()),
), __GLk()),
("> -- Author Name", (
sy.BlockQuoteAuthorToken(["Author Name"]),
), __GL()),
), __GLk()),
],
"code": [
("```\nText\n```", (sy.CodeToken(["```"]),), __GL()),
("```\nText\nText\n```", (sy.CodeToken(["```"]),), __GL()),
("```\nText\n```", (sy.CodeToken(["```"]),), __GLk()),
("```\nText\nText\n```", (sy.CodeToken(["```"]),), __GLk()),
],
}
@ -666,6 +668,7 @@ list_of_text_input_and_readable_output = [
# Other Mostly for testing and coverage
("'", "quotes", "'"),
(" -", "line", "-"),
]
@ -824,7 +827,7 @@ def test_advanced_expression_and_token_creation(markup_element, to_parse, expect
result = expr.parse_string(to_parse)
print()
print(f"Parsing string: '{to_parse}'")
print("Defined at %(filename)s:%(lineno)d" % {'filename': __file__, 'lineno': line_test})
print(line_test)
print(f"With expression: '{markup_element}'")
print("Defined at %(filename)s:%(lineno)d" % {'filename': __syntax_file,
'lineno': __definition_of_syntax_elements[markup_element]})

View File

@ -47,6 +47,7 @@ def test_counterpart_container(type_n, expected_counterpart, expected_container,
try:
token = type_n(["test"]) # noqa
token.line_number = 1
token.index = 1
assert token.counterpart() == expected_counterpart
if expected_container == "error":
with pytest.raises(error_mngr.MismatchedContainerError):

View File

@ -1,7 +1,7 @@
import pytest
from bootstraparse.modules import syntax, error_mngr, tools, context_mngr
from bootstraparse.modules.tools import __GL, __module_path, find_variables_in_file
from bootstraparse.modules import syntax # , error_mngr, tools, context_mngr
# from bootstraparse.modules.tools import __GL, __module_path, find_variables_in_file
__XF = pytest.mark.xfail