From ac600bb86ece1530cac00bbe418e8b1225c975fd Mon Sep 17 00:00:00 2001 From: Bryce Willey Date: Mon, 11 May 2026 09:41:08 -0400 Subject: [PATCH 01/13] Bump Flask.SocketIO version to 5.6.1 Needed because 5.6.0 is broken on Flask versions 3.1.2 and up (docassemble is currently using 3.1.3). An example of the errors that appear in websockets.log ``` File "/usr/share/docassemble/local3.12/lib/python3.12/site-packages/flask_socketio/__init__.py", line 815, in _handle_event ctx.session = session_obj ^^^^^^^^^^^ AttributeError: property 'session' of 'RequestContext' object has no setter ``` Full description of what's fixed at https://github.com/miguelgrinberg/Flask-SocketIO/issues/2152. --- docassemble_webapp/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docassemble_webapp/pyproject.toml b/docassemble_webapp/pyproject.toml index d6a7dba6b..c2081c315 100644 --- a/docassemble_webapp/pyproject.toml +++ b/docassemble_webapp/pyproject.toml @@ -83,7 +83,7 @@ dependencies = [ "flask-cors==6.0.2", "Flask-Login==0.6.3", "Flask-Mail==0.10.0", - "Flask-SocketIO==5.6.0", + "Flask-SocketIO==5.6.1", "Flask-SQLAlchemy==3.1.1", "Flask-WTF==1.2.2", "fonttools==4.61.1", From f86e9d75e9967f93f04843331526aeea41399010 Mon Sep 17 00:00:00 2001 From: Jonathan Pyle Date: Mon, 11 May 2026 21:50:50 -0400 Subject: [PATCH 02/13] changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9db31e848..71c98a656 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Change Log +## [1.9.12] - 2026-05-11 + +### Fixed + +- Issue with `Flask.SocketIO`. + ## [1.9.11] - 2026-05-08 ### Fixed From daf98ba12dd3eacbac3d4eaefd7281abfb90e13e Mon Sep 17 00:00:00 2001 From: Jonathan Pyle Date: Mon, 11 May 2026 22:33:40 -0400 Subject: [PATCH 03/13] =?UTF-8?q?Bump=20version:=201.9.11=20=E2=86=92=201.?= =?UTF-8?q?9.12?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- Docker/VERSION | 2 +- docassemble_base/docassemble/base/__init__.py | 2 +- docassemble_base/pyproject.toml | 2 +- docassemble_demo/docassemble/demo/__init__.py | 2 +- docassemble_demo/pyproject.toml | 4 ++-- docassemble_webapp/docassemble/webapp/__init__.py | 2 +- docassemble_webapp/docassemble/webapp/data/VERSION.txt | 2 +- docassemble_webapp/docassemble/webapp/setup.py | 2 +- docassemble_webapp/pyproject.toml | 6 +++--- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ea1e99dc2..30c8d4ead 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.9.11 +current_version = 1.9.12 commit = True tag = True diff --git a/Docker/VERSION b/Docker/VERSION index 5e9287b86..1fe2d3795 100644 --- a/Docker/VERSION +++ b/Docker/VERSION @@ -1 +1 @@ -1.9.11 +1.9.12 diff --git a/docassemble_base/docassemble/base/__init__.py b/docassemble_base/docassemble/base/__init__.py index b95b03c16..c413c32a6 100644 --- a/docassemble_base/docassemble/base/__init__.py +++ b/docassemble_base/docassemble/base/__init__.py @@ -1 +1 @@ -__version__ = "1.9.11" +__version__ = "1.9.12" diff --git a/docassemble_base/pyproject.toml b/docassemble_base/pyproject.toml index c3eb8a979..f073a0ba6 100644 --- a/docassemble_base/pyproject.toml +++ b/docassemble_base/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools==82.0.1"] build-backend = "setuptools.build_meta" [project] name = "docassemble.base" -version = '1.9.11' +version = '1.9.12' description = "The base components of the docassemble system." readme = "README.md" authors = [ diff --git a/docassemble_demo/docassemble/demo/__init__.py b/docassemble_demo/docassemble/demo/__init__.py index 5928e6b57..fbdb3385b 100644 --- a/docassemble_demo/docassemble/demo/__init__.py +++ b/docassemble_demo/docassemble/demo/__init__.py @@ -1,2 +1,2 @@ # do not pre-load -__version__ = "1.9.11" +__version__ = "1.9.12" diff --git a/docassemble_demo/pyproject.toml b/docassemble_demo/pyproject.toml index 16b8d8796..7fb81713d 100644 --- a/docassemble_demo/pyproject.toml +++ b/docassemble_demo/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools==82.0.1"] build-backend = "setuptools.build_meta" [project] name = "docassemble.demo" -version = '1.9.11' +version = '1.9.12' description = "A demonstration package for docassemble." readme = "README.md" authors = [ @@ -13,7 +13,7 @@ license = "MIT" license-files = ["LICENSE.txt"] requires-python = ">= 3.9" dependencies = [ - 'docassemble.base==1.9.11' + 'docassemble.base==1.9.12' # "ics", # "openai", # "anthropic", diff --git a/docassemble_webapp/docassemble/webapp/__init__.py b/docassemble_webapp/docassemble/webapp/__init__.py index b95b03c16..c413c32a6 100644 --- a/docassemble_webapp/docassemble/webapp/__init__.py +++ b/docassemble_webapp/docassemble/webapp/__init__.py @@ -1 +1 @@ -__version__ = "1.9.11" +__version__ = "1.9.12" diff --git a/docassemble_webapp/docassemble/webapp/data/VERSION.txt b/docassemble_webapp/docassemble/webapp/data/VERSION.txt index 5e9287b86..1fe2d3795 100644 --- a/docassemble_webapp/docassemble/webapp/data/VERSION.txt +++ b/docassemble_webapp/docassemble/webapp/data/VERSION.txt @@ -1 +1 @@ -1.9.11 +1.9.12 diff --git a/docassemble_webapp/docassemble/webapp/setup.py b/docassemble_webapp/docassemble/webapp/setup.py index 2a732b7d6..e91e5222f 100644 --- a/docassemble_webapp/docassemble/webapp/setup.py +++ b/docassemble_webapp/docassemble/webapp/setup.py @@ -4,7 +4,7 @@ from docassemble.webapp.app_object import app from docassemble.base.config import daconfig import docassemble.webapp.database -da_version = '1.9.11' +da_version = '1.9.12' app.config['DA_VERSION'] = da_version app.config['APP_NAME'] = daconfig.get('appname', 'docassemble') app.config['BRAND_NAME'] = daconfig.get('brandname', daconfig.get('appname', 'docassemble')) diff --git a/docassemble_webapp/pyproject.toml b/docassemble_webapp/pyproject.toml index c2081c315..d27de9453 100644 --- a/docassemble_webapp/pyproject.toml +++ b/docassemble_webapp/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools==82.0.1"] build-backend = "setuptools.build_meta" [project] name = "docassemble.webapp" -version = '1.9.11' +version = '1.9.12' description = "The web application components of the docassemble system." readme = "README.md" authors = [ @@ -13,8 +13,8 @@ license = "MIT" license-files = ["LICENSE.txt"] requires-python = ">= 3.9" dependencies = [ - 'docassemble.base==1.9.11', - 'docassemble.demo==1.9.11', + 'docassemble.base==1.9.12', + 'docassemble.demo==1.9.12', "aiohappyeyeballs==2.6.1", "aiohttp==3.13.5", "aiohttp-retry==2.9.1", From 186e8e7a8b2d349e04d98b2b1a19bf0f172f84ac Mon Sep 17 00:00:00 2001 From: Jonathan Pyle Date: Mon, 18 May 2026 05:57:26 -0400 Subject: [PATCH 04/13] upgrade certifi --- docassemble_base/pyproject.toml | 2 +- docassemble_webapp/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docassemble_base/pyproject.toml b/docassemble_base/pyproject.toml index f073a0ba6..8678fd93d 100644 --- a/docassemble_base/pyproject.toml +++ b/docassemble_base/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "cachetools==5.5.2", "cairocffi==1.7.1", "CairoSVG==2.8.2", - "certifi==2025.11.12", + "certifi==2026.2.25", "cffi==2.0.0", "chardet==5.2.0", "charset-normalizer==3.4.4", diff --git a/docassemble_webapp/pyproject.toml b/docassemble_webapp/pyproject.toml index d27de9453..a57abf6db 100644 --- a/docassemble_webapp/pyproject.toml +++ b/docassemble_webapp/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "cairocffi==1.7.1", "CairoSVG==2.8.2", "celery==5.6.0", - "certifi==2025.11.12", + "certifi==2026.2.25", "cffi==2.0.0", "chardet==5.2.0", "charset-normalizer==3.4.4", From b9f53e42ab60b248b25a42187532f4988747a2aa Mon Sep 17 00:00:00 2001 From: Jonathan Pyle Date: Sun, 24 May 2026 21:02:50 -0400 Subject: [PATCH 05/13] update unfocus step in tests --- tests/features/steps/docassemble.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/features/steps/docassemble.py b/tests/features/steps/docassemble.py index f6020a867..46f46b0ef 100644 --- a/tests/features/steps/docassemble.py +++ b/tests/features/steps/docassemble.py @@ -613,9 +613,6 @@ def change_window_size(context, xdimen, ydimen): @step(r'I unfocus') def unfocus(context): - elem = WebDriverWait(context.browser, 10).until( - EC.element_to_be_clickable((By.ID, 'daMainQuestion')) - ) context.browser.execute_script("document.activeElement.blur();") From b15716dc3cdcdcfda52051c0046cefeaacb9ba34 Mon Sep 17 00:00:00 2001 From: Jonathan Pyle Date: Wed, 3 Jun 2026 21:09:16 -0400 Subject: [PATCH 06/13] incorrect ghostscript command --- CHANGELOG.md | 6 ++++++ docassemble_base/docassemble/base/pdfa.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71c98a656..24e853fd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Change Log +## [1.9.13] - 2026-06-03 + +### Fixed + +- Issue with converting fillable PDFs to PDF/A. + ## [1.9.12] - 2026-05-11 ### Fixed diff --git a/docassemble_base/docassemble/base/pdfa.py b/docassemble_base/docassemble/base/pdfa.py index 9382f4a2e..c24b10fcf 100644 --- a/docassemble_base/docassemble/base/pdfa.py +++ b/docassemble_base/docassemble/base/pdfa.py @@ -8,7 +8,7 @@ def pdf_to_pdfa(filename): outfile = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) directory = tempfile.mkdtemp(prefix='SavedFile') - commands = ['gs', '-dPDFA', '-dBATCH', '-dNOPAUSE', '-sColorConversionStrategy=UseDeviceIndependentColor', '-sProcessColorModel=DeviceCMYK', '-sDEVICE=pdfwrite', '-sPDFACompatibilityPolicy=1', '-sOutputFile=' + outfile.name, filename] + commands = ['gs', '-dPDFA=2', '-dBATCH', '-dNOPAUSE', '-sColorConversionStrategy=CMYK', '-sProcessColorModel=DeviceCMYK', '-sDEVICE=pdfwrite', '-dPDFACompatibilityPolicy=1', '-sOutputFile=' + outfile.name, filename] try: output = subprocess.check_output(commands, cwd=directory, stderr=subprocess.STDOUT).decode() except subprocess.CalledProcessError as err: From b339d530e151fcc585b881165be1cc5793d04060 Mon Sep 17 00:00:00 2001 From: Jonathan Pyle Date: Wed, 3 Jun 2026 22:47:42 -0400 Subject: [PATCH 07/13] =?UTF-8?q?Bump=20version:=201.9.12=20=E2=86=92=201.?= =?UTF-8?q?9.13?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- Docker/VERSION | 2 +- docassemble_base/docassemble/base/__init__.py | 2 +- docassemble_base/pyproject.toml | 2 +- docassemble_demo/docassemble/demo/__init__.py | 2 +- docassemble_demo/pyproject.toml | 4 ++-- docassemble_webapp/docassemble/webapp/__init__.py | 2 +- docassemble_webapp/docassemble/webapp/data/VERSION.txt | 2 +- docassemble_webapp/docassemble/webapp/setup.py | 2 +- docassemble_webapp/pyproject.toml | 6 +++--- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 30c8d4ead..07b7f1123 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.9.12 +current_version = 1.9.13 commit = True tag = True diff --git a/Docker/VERSION b/Docker/VERSION index 1fe2d3795..2a72e6f9f 100644 --- a/Docker/VERSION +++ b/Docker/VERSION @@ -1 +1 @@ -1.9.12 +1.9.13 diff --git a/docassemble_base/docassemble/base/__init__.py b/docassemble_base/docassemble/base/__init__.py index c413c32a6..2070dea4e 100644 --- a/docassemble_base/docassemble/base/__init__.py +++ b/docassemble_base/docassemble/base/__init__.py @@ -1 +1 @@ -__version__ = "1.9.12" +__version__ = "1.9.13" diff --git a/docassemble_base/pyproject.toml b/docassemble_base/pyproject.toml index 8678fd93d..2d0fe6b3f 100644 --- a/docassemble_base/pyproject.toml +++ b/docassemble_base/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools==82.0.1"] build-backend = "setuptools.build_meta" [project] name = "docassemble.base" -version = '1.9.12' +version = '1.9.13' description = "The base components of the docassemble system." readme = "README.md" authors = [ diff --git a/docassemble_demo/docassemble/demo/__init__.py b/docassemble_demo/docassemble/demo/__init__.py index fbdb3385b..1ceced84e 100644 --- a/docassemble_demo/docassemble/demo/__init__.py +++ b/docassemble_demo/docassemble/demo/__init__.py @@ -1,2 +1,2 @@ # do not pre-load -__version__ = "1.9.12" +__version__ = "1.9.13" diff --git a/docassemble_demo/pyproject.toml b/docassemble_demo/pyproject.toml index 7fb81713d..dff8674a0 100644 --- a/docassemble_demo/pyproject.toml +++ b/docassemble_demo/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools==82.0.1"] build-backend = "setuptools.build_meta" [project] name = "docassemble.demo" -version = '1.9.12' +version = '1.9.13' description = "A demonstration package for docassemble." readme = "README.md" authors = [ @@ -13,7 +13,7 @@ license = "MIT" license-files = ["LICENSE.txt"] requires-python = ">= 3.9" dependencies = [ - 'docassemble.base==1.9.12' + 'docassemble.base==1.9.13' # "ics", # "openai", # "anthropic", diff --git a/docassemble_webapp/docassemble/webapp/__init__.py b/docassemble_webapp/docassemble/webapp/__init__.py index c413c32a6..2070dea4e 100644 --- a/docassemble_webapp/docassemble/webapp/__init__.py +++ b/docassemble_webapp/docassemble/webapp/__init__.py @@ -1 +1 @@ -__version__ = "1.9.12" +__version__ = "1.9.13" diff --git a/docassemble_webapp/docassemble/webapp/data/VERSION.txt b/docassemble_webapp/docassemble/webapp/data/VERSION.txt index 1fe2d3795..2a72e6f9f 100644 --- a/docassemble_webapp/docassemble/webapp/data/VERSION.txt +++ b/docassemble_webapp/docassemble/webapp/data/VERSION.txt @@ -1 +1 @@ -1.9.12 +1.9.13 diff --git a/docassemble_webapp/docassemble/webapp/setup.py b/docassemble_webapp/docassemble/webapp/setup.py index e91e5222f..95fdee9fa 100644 --- a/docassemble_webapp/docassemble/webapp/setup.py +++ b/docassemble_webapp/docassemble/webapp/setup.py @@ -4,7 +4,7 @@ from docassemble.webapp.app_object import app from docassemble.base.config import daconfig import docassemble.webapp.database -da_version = '1.9.12' +da_version = '1.9.13' app.config['DA_VERSION'] = da_version app.config['APP_NAME'] = daconfig.get('appname', 'docassemble') app.config['BRAND_NAME'] = daconfig.get('brandname', daconfig.get('appname', 'docassemble')) diff --git a/docassemble_webapp/pyproject.toml b/docassemble_webapp/pyproject.toml index a57abf6db..a7bc9835c 100644 --- a/docassemble_webapp/pyproject.toml +++ b/docassemble_webapp/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools==82.0.1"] build-backend = "setuptools.build_meta" [project] name = "docassemble.webapp" -version = '1.9.12' +version = '1.9.13' description = "The web application components of the docassemble system." readme = "README.md" authors = [ @@ -13,8 +13,8 @@ license = "MIT" license-files = ["LICENSE.txt"] requires-python = ">= 3.9" dependencies = [ - 'docassemble.base==1.9.12', - 'docassemble.demo==1.9.12', + 'docassemble.base==1.9.13', + 'docassemble.demo==1.9.13', "aiohappyeyeballs==2.6.1", "aiohttp==3.13.5", "aiohttp-retry==2.9.1", From f62b67e08b88a375591e2d0d65ce12d6d6d8c4d9 Mon Sep 17 00:00:00 2001 From: Jonathan Pyle Date: Thu, 25 Jun 2026 20:20:56 -0400 Subject: [PATCH 08/13] reorganized code; use blueprints with configuration directives for disabling features; use pluggy; use signatures for celery; fix issue with hidden fields inside of list collect --- .bumpversion.cfg | 2 - .gitignore | 9 +- CHANGELOG.md | 27 + Docker/config/config.yml.dist | 10 + Docker/cron/docassemble-cron-daily.sh | 20 +- Docker/initialize.sh | 162 +- Docker/reset.sh | 8 +- Docker/restart-post-logrotate.sh | 2 +- Docker/run-apache.sh | 2 +- Docker/run-cron.sh | 4 +- Dockerfile | 6 +- LICENSE.txt | 2 +- docassemble_base/LICENSE.txt | 2 +- docassemble_base/config.yml | 10 + docassemble_base/docassemble/base/DA.py | 1 + docassemble_base/docassemble/base/amazon.py | 10 +- .../docassemble/base/astparser.py | 21 +- .../docassemble/base/background.py | 63 + docassemble_base/docassemble/base/config.py | 25 +- docassemble_base/docassemble/base/core.py | 32 +- .../base/data/sources/base-words.yml | 7 +- docassemble_base/docassemble/base/dates.py | 484 + docassemble_base/docassemble/base/empty.py | 199 + docassemble_base/docassemble/base/error.py | 2 +- .../docassemble/base/file_docx.py | 654 +- docassemble_base/docassemble/base/filter.py | 2006 -- .../docassemble/base/filter}/__init__.py | 0 .../docassemble/base/filter/docx.py | 803 + .../docassemble/base/filter/docx_subdoc.py | 39 + .../docassemble/base/filter/html.py | 959 + .../docassemble/base/filter/image_docx.py | 56 + .../docassemble/base/filter/pandoc.py | 710 + .../docassemble/base/filter/utils.py | 131 + .../docassemble/base/functions.py | 3149 +- docassemble_base/docassemble/base/geocode.py | 14 +- docassemble_base/docassemble/base/helpers.py | 29 + docassemble_base/docassemble/base/hooks.py | 553 + .../docassemble/base/hookspecs.py | 506 + .../docassemble/base/interview_cache.py | 12 +- .../docassemble/base/interview_source.py | 311 + docassemble_base/docassemble/base/jinja.py | 570 + .../docassemble/base/language}/__init__.py | 0 .../base/language/capitalization.py | 17 + .../docassemble/base/language/control.py | 146 + .../docassemble/base/language/core.py | 38 + .../docassemble/base/language/currency.py | 125 + .../docassemble/base/language/language.py | 747 + .../docassemble/base/language/language_de.py | 74 + .../docassemble/base/language/language_en.py | 162 + .../docassemble/base/language/language_es.py | 72 + .../docassemble/base/language/language_fr.py | 65 + .../docassemble/base/language/language_it.py | 65 + .../docassemble/base/language/language_nl.py | 65 + .../docassemble/base/language/numbers.py | 203 + .../docassemble/base/language/utils.py | 29 + .../docassemble/base/language/words.py | 75 + docassemble_base/docassemble/base/legal.py | 281 +- docassemble_base/docassemble/base/mako/ast.py | 4 +- .../docassemble/base/mako/exceptions.py | 4 +- .../docassemble/base/microsoft.py | 21 +- docassemble_base/docassemble/base/pandoc.py | 102 +- docassemble_base/docassemble/base/parse.py | 2105 +- .../docassemble/base/pattern_server.py | 12 +- docassemble_base/docassemble/base/pdfa.py | 1 - docassemble_base/docassemble/base/pdftk.py | 10 +- .../docassemble/base/plugin_manager.py | 32 + docassemble_base/docassemble/base/py.typed | 0 .../docassemble/base/read_config.py | 14 +- docassemble_base/docassemble/base/sql.py | 91 +- .../docassemble/base/standardformatter.py | 186 +- .../docassemble/base/thread_context.py | 157 + docassemble_base/docassemble/base/util.py | 2090 +- docassemble_base/py.typed | 0 docassemble_base/pyproject.toml | 60 +- docassemble_demo/LICENSE.txt | 2 +- docassemble_demo/docassemble/demo/aihelper.py | 2 +- .../docassemble/demo/alembic/env.py | 2 +- docassemble_demo/docassemble/demo/calendar.py | 12 +- .../docassemble/demo/change_suffix.py | 5 +- .../docassemble/demo/chatbot_bg.py | 4 +- .../docassemble/demo/chatbot_bg_object.py | 2 +- .../docassemble/demo/companydb.py | 22 +- .../docassemble/demo/contexttest.py | 2 +- docassemble_demo/docassemble/demo/custombg.py | 11 +- .../data/questions/examples/stage-one.yml | 2 +- .../demo/data/templates/myinterviews.html | 6 +- docassemble_demo/docassemble/demo/demodb.py | 11 +- docassemble_demo/docassemble/demo/ein.py | 2 +- .../docassemble/demo/fruit_database.py | 2 +- docassemble_demo/docassemble/demo/gd_fetch.py | 4 +- docassemble_demo/docassemble/demo/gettime.py | 4 +- .../docassemble/demo/google_drive.py | 22 +- .../docassemble/demo/google_sheets.py | 5 +- .../docassemble/demo/google_vision2.py | 6 +- docassemble_demo/docassemble/demo/index.py | 8 +- .../docassemble/demo/oauthsheets.py | 11 +- docassemble_demo/docassemble/demo/py.typed | 0 .../demo/{random-test.py => random_test.py} | 23 +- .../docassemble/demo/reldemodb.py | 8 +- docassemble_demo/docassemble/demo/sign.py | 38 +- docassemble_demo/docassemble/demo/ssn.py | 8 +- .../docassemble/demo/testcustombg.py | 14 +- .../docassemble/demo/translate.py | 4 +- .../docassemble/demo/user_permissions.py | 35 +- docassemble_demo/py.typed | 0 docassemble_demo/pyproject.toml | 47 + docassemble_webapp/LICENSE.txt | 2 +- docassemble_webapp/MANIFEST.in | 11 + .../docassemble/webapp/admin/__init__.py | 1 + .../docassemble/webapp/admin/forms.py | 21 + .../docassemble/webapp/admin/funcs.py | 59 + .../templates/admin}/config.html | 0 .../templates/admin}/interviews.html | 6 +- .../templates/admin}/start-embedded.html | 0 .../templates/admin}/start.html | 0 .../docassemble/webapp/admin/views.py | 321 + .../docassemble/webapp/alembic/env.py | 10 +- .../025e06b85efc_package_github_branch.py | 1 + .../versions/4328f2c08f05_add_indexes.py | 1 + ...f71cf543a4_temp_user_id_to_userdictkeys.py | 1 + .../versions/693d02299f38_fix_indexes.py | 1 + .../77e8971ffcbf_first_alembic_revision.py | 1 + ..._alter_database_for_mysql_compatibility.py | 1 + .../a8746ddab8cb_add_unique_constraints.py | 1 + .../versions/c2ef4831ef76_user_last_login.py | 1 + ...eb61567ea005_length_of_subdivisionfirst.py | 1 + .../f0b00081fda9_uploads_persistent_global.py | 1 + ...c46_add_a_column_for_the_voicerss_voice.py | 1 + .../docassemble/webapp/api/__init__.py | 2 + .../docassemble/webapp/api/api.py | 228 + .../docassemble/webapp/api/blueprint.py | 7 + .../docassemble/webapp/api/forms.py | 40 + .../docassemble/webapp/api/helpers.py | 264 + .../templates/api}/manage_api.html | 8 +- .../docassemble/webapp/api/views.py | 179 + .../docassemble/webapp/app_initialize.py | 269 + .../docassemble/webapp/app_object.py | 54 +- .../docassemble/webapp/app_socket.py | 52 - .../docassemble/webapp/auth/__init__.py | 1 + .../docassemble/webapp/auth/auth0.py | 49 + .../docassemble/webapp/auth/authentik.py | 60 + .../docassemble/webapp/auth/azure.py | 46 + .../docassemble/webapp/auth/blueprint.py | 7 + .../docassemble/webapp/auth/facebook.py | 43 + .../docassemble/webapp/auth/google.py | 59 + .../docassemble/webapp/auth/helpers.py | 4 + .../docassemble/webapp/auth/keycloak.py | 64 + .../docassemble/webapp/auth/miniorange.py | 55 + .../docassemble/webapp/auth/oauth.py | 35 + .../webapp/auth/templates/__init__.py | 0 .../templates/auth}/google_login.html | 2 +- .../docassemble/webapp/auth/views.py | 122 + .../docassemble/webapp/auth/zitadel.py | 58 + .../docassemble/webapp/backend.py | 1368 - .../docassemble/webapp/cleanup_sessions.py | 23 +- .../docassemble/webapp/cloud.py | 27 - .../docassemble/webapp/cloud/utils.py | 55 + .../docassemble/webapp/cloud_deregister.py | 8 +- .../docassemble/webapp/cloud_register.py | 11 +- .../docassemble/webapp/config.py | 240 + .../docassemble/webapp/config_worker.py | 6 +- .../docassemble/webapp/core/__init__.py | 4 - .../docassemble/webapp/core/models.py | 143 - .../docassemble/webapp/create_tables.py | 357 +- .../docassemble/webapp/createminio.py | 6 +- docassemble_webapp/docassemble/webapp/cron.py | 302 +- .../docassemble/webapp/cron_tasks/__init__.py | 1 + .../webapp/cron_tasks/blueprint.py | 7 + .../docassemble/webapp/cron_tasks/cli.py | 311 + .../docassemble/webapp/daglobal/__init__.py | 0 .../docassemble/webapp/daglobal/helpers.py | 67 + .../docassemble/webapp/daglobal/hooks.py | 80 + .../docassemble/webapp/daglobal/models.py | 20 + .../docassemble/webapp/daredis.py | 2 +- .../docassemble/webapp/database.py | 9 +- docassemble_webapp/docassemble/webapp/db.py | 55 + .../docassemble/webapp/db_base.py | 8 + .../docassemble/webapp/db_object.py | 57 - .../docassemble/webapp/deregister.py | 23 +- .../docassemble/webapp/develop/__init__.py | 1 + .../docassemble/webapp/develop/api.py | 479 + .../docassemble/webapp/develop/blueprint.py | 7 + .../docassemble/webapp/develop/common.py | 56 + .../webapp/{develop.py => develop/forms.py} | 114 +- .../docassemble/webapp/develop/helpers.py | 334 + .../docassemble/webapp/develop/oauth.py | 36 + .../templates/develop}/create_package.html | 0 .../develop}/create_playground_package.html | 4 +- .../templates/develop}/gd_sync_wait.html | 0 .../templates/develop}/github.html | 0 .../templates/develop}/google_sync.html | 0 .../templates/develop}/googledrive.html | 0 .../develop}/manage_playgrounds.html | 4 +- .../templates/develop}/manage_projects.html | 20 +- .../templates/develop}/od_sync_wait.html | 0 .../templates/develop}/officeaddin.html | 0 .../develop}/officefunctionfile.html | 0 .../templates/develop}/officemanifest.xml | 0 .../templates/develop}/officeouter.html | 4 +- .../templates/develop}/onedrive.html | 0 .../templates/develop}/playground.html | 32 +- .../templates/develop}/playground_poll.html | 0 .../templates/develop}/playgroundfiles.html | 20 +- .../develop}/playgroundpackages.html | 16 +- .../develop}/pull_playground_package.html | 2 +- .../templates/develop}/rename_project.html | 4 +- .../templates/develop}/test_embed.html | 0 .../develop}/update_package_wait.html | 0 .../templates/develop}/utilities.html | 8 +- .../templates/develop}/view_source.html | 0 .../docassemble/webapp/develop/views.py | 5496 ++++ .../webapp/emailserver/__init__.py | 0 .../docassemble/webapp/emailserver/helpers.py | 225 + .../docassemble/webapp/emailserver/models.py | 58 + .../docassemble/webapp/errors.py | 311 + .../docassemble/webapp/extensions.py | 16 + .../docassemble/webapp/fax/__init__.py | 1 + .../docassemble/webapp/{ => fax}/clicksend.py | 18 +- .../docassemble/webapp/fax/helpers.py | 128 + .../docassemble/webapp/{ => fax}/telnyx.py | 17 +- .../docassemble/webapp/fax/views.py | 109 + .../docassemble/webapp/file_number.py | 9 - .../docassemble/webapp/files/__init__.py | 1 + .../docassemble/webapp/files/api.py | 53 + .../docassemble/webapp/files/blueprint.py | 6 + .../docassemble/webapp/files/common.py | 34 + .../webapp/{ => files}/file_access.py | 56 +- .../docassemble/webapp/files/file_number.py | 10 + .../docassemble/webapp/files/helpers.py | 334 + .../docassemble/webapp/files/hooks.py | 28 + .../webapp/{ => files}/package_info.py | 4 +- .../webapp/{files.py => files/savedfile.py} | 92 +- .../docassemble/webapp/files/views.py | 73 + .../webapp/fix_postgresql_tables.py | 22 +- .../docassemble/webapp/fixup.sh | 27 + .../docassemble/webapp/flask_app.py | 4 + .../docassemble/webapp/flask_sql_config.py | 81 + .../docassemble/webapp/google_api.py | 22 +- .../docassemble/webapp/hooks/__init__.py | 0 .../docassemble/webapp/hooks/default.py | 480 + .../docassemble/webapp/hooks/impl.py | 3 + .../docassemble/webapp/install_certs.py | 19 +- .../docassemble/webapp/interview/__init__.py | 1 + .../docassemble/webapp/interview/api.py | 632 + .../docassemble/webapp/interview/blueprint.py | 6 + .../docassemble/webapp/interview/common.py | 22 + .../docassemble/webapp/interview/config.py | 87 + .../webapp/interview/dictionary.py | 23 + .../docassemble/webapp/interview/helpers.py | 1748 ++ .../docassemble/webapp/interview/hooks.py | 6 + .../docassemble/webapp/interview/models.py | 43 + .../docassemble/webapp/interview/views.py | 3409 +++ .../webapp/jsonstorage/__init__.py | 0 .../{jsonstore.py => jsonstorage/helpers.py} | 34 +- .../docassemble/webapp/jsonstorage/models.py | 23 + .../docassemble/webapp/lifecycle.py | 73 + .../docassemble/webapp/list-cloud.py | 5 +- .../docassemble/webapp/listlog.py | 12 +- docassemble_webapp/docassemble/webapp/lock.py | 74 + .../docassemble/webapp/log_initialize.py | 86 + .../docassemble/webapp/logs/__init__.py | 1 + .../docassemble/webapp/logs/forms.py | 12 + .../pages => logs/templates/logs}/logs.html | 10 +- .../docassemble/webapp/logs/views.py | 161 + .../docassemble/webapp/mail/__init__.py | 0 .../webapp/{ => mail}/azure_mail.py | 16 +- .../webapp/{ => mail}/da_flask_mail.py | 7 +- .../docassemble/webapp/mail/helpers.py | 102 + .../docassemble/webapp/mail/hooks.py | 17 + .../webapp/{ => mail}/mailgun_mail.py | 14 +- .../webapp/{ => mail}/sendgrid_mail.py | 30 +- .../docassemble/webapp/main/__init__.py | 1 + .../docassemble/webapp/main/api.py | 254 + .../docassemble/webapp/main/blueprint.py | 6 + .../docassemble/webapp/main/helpers.py | 348 + .../docassemble/webapp/main/hooks.py | 99 + .../docassemble/webapp/main/models.py | 71 + .../templates/main}/restart.html | 0 .../docassemble/webapp/main/views.py | 423 + .../docassemble/webapp/ml/__init__.py | 1 + .../docassemble/webapp/ml/blueprint.py | 7 + .../docassemble/webapp/ml/common.py | 6 + .../docassemble/webapp/ml/config.py | 17 + .../docassemble/webapp/ml/forms.py | 18 + .../docassemble/webapp/ml/helpers.py | 21 + .../docassemble/webapp/ml/hooks.py | 108 + .../webapp/{ => ml}/machinelearning.py | 26 +- .../docassemble/webapp/ml/models.py | 18 + .../pages => ml/templates/ml}/train.html | 54 +- .../docassemble/webapp/ml/views.py | 404 + .../docassemble/webapp/monitor/__init__.py | 1 + .../docassemble/webapp/monitor/blueprint.py | 7 + .../docassemble/webapp/monitor/hooks.py | 261 + .../docassemble/webapp/monitor/models.py | 33 + .../templates/monitor}/monitor.html | 0 .../docassemble/webapp/monitor/views.py | 322 + .../webapp/objectstore/__init__.py | 0 .../docassemble/webapp/objectstore/hooks.py | 26 + .../docassemble/webapp/objectstore/models.py | 16 + .../docassemble/webapp/packages/__init__.py | 1 + .../docassemble/webapp/packages/api.py | 247 + .../docassemble/webapp/packages/blueprint.py | 7 + .../docassemble/webapp/packages/forms.py | 16 + .../docassemble/webapp/packages/helpers.py | 135 + .../docassemble/webapp/packages/models.py | 73 +- .../docassemble/webapp/packages/pip.py | 18 + .../templates/packages}/update_package.html | 6 +- .../docassemble/webapp/packages/views.py | 225 + .../docassemble/webapp/phonelogin/__init__.py | 1 + .../docassemble/webapp/phonelogin/forms.py | 48 + .../templates/phonelogin}/phone_login.html | 0 .../phonelogin}/phone_login_verify.html | 0 .../docassemble/webapp/phonelogin/views.py | 157 + .../docassemble/webapp/playground.py | 71 +- .../docassemble/webapp/process_email.py | 178 +- .../docassemble/webapp/py.typed | 0 .../docassemble/webapp/react/__init__.py | 1 + .../docassemble/webapp/react/api.py | 411 + .../docassemble/webapp/register.py | 30 +- .../docassemble/webapp/restart.py | 7 +- .../docassemble/webapp/screenreader.py | 5 +- .../docassemble/webapp/server.py | 24495 +--------------- .../webapp/{ => services}/validators.py | 2 +- .../docassemble/webapp/sessions.py | 119 + .../docassemble/webapp/setup.py | 309 +- .../docassemble/webapp/signals.py | 4 + .../docassemble/webapp/sms/__init__.py | 1 + .../docassemble/webapp/sms/helpers.py | 102 + .../docassemble/webapp/sms/views.py | 834 + .../docassemble/webapp/socketserver.py | 150 +- .../docassemble/webapp/sqlalchemy_url.py | 2 +- .../docassemble/webapp/tasks/__init__.py | 0 .../docassemble/webapp/tasks/app.py | 3 + .../docassemble/webapp/tasks/app_object.py | 19 + .../docassemble/webapp/tasks/common.py | 144 + .../docassemble/webapp/tasks/context.py | 15 + .../docassemble/webapp/tasks/hooks.py | 16 + .../{worker_tasks.py => tasks/tasks.py} | 636 +- .../webapp/templates/base_templates/base.html | 38 +- .../templates/flask_user/change_password.html | 2 +- .../webapp/templates/flask_user/invite.html | 2 +- .../webapp/templates/flask_user/login.html | 22 +- .../webapp/templates/flask_user/register.html | 22 +- .../webapp/templates/pages/501.html | 2 +- .../webapp/templates/pages/admin_page.html | 28 - .../webapp/templates/pages/home_page.html | 36 - .../webapp/templates/pages/member_page.html | 28 - .../webapp/templates/pages/packages.html | 19 - .../webapp/templates/pages/socketserver.html | 9 - .../templates/pages/testgoogledrive.html | 7 - .../webapp/templates/pages/testpost.html | 21 - .../webapp/templates/pages/webrtc.html | 26 - .../docassemble/webapp/testing.py | 33 + .../webapp/translation/__init__.py | 1 + .../docassemble/webapp/translation/views.py | 571 + .../docassemble/webapp/translations.py | 6 +- .../docassemble/webapp/tts/__init__.py | 1 + .../docassemble/webapp/tts/hooks.py | 77 + .../docassemble/webapp/tts/models.py | 32 + .../docassemble/webapp/tts/views.py | 103 + .../docassemble/webapp/twilio/__init__.py | 0 .../docassemble/webapp/twilio/helpers.py | 34 + .../docassemble/webapp/update.py | 1063 +- .../docassemble/webapp/update_config.py | 11 +- .../docassemble/webapp/user_database.py | 7 +- .../docassemble/webapp/users/__init__.py | 1 + .../docassemble/webapp/users/api.py | 440 + .../docassemble/webapp/users/blueprint.py | 7 + .../docassemble/webapp/users/common.py | 23 + .../docassemble/webapp/users/forms.py | 163 +- .../docassemble/webapp/users/helpers.py | 476 + .../docassemble/webapp/users/hooks.py | 26 + .../docassemble/webapp/users/loginurl.py | 97 + .../docassemble/webapp/users/models.py | 164 +- .../docassemble/webapp/users/signals.py | 52 + .../webapp/users/templates/users/__init__.py | 0 .../templates/users/add_user_page.html | 2 +- .../users/edit_user_profile_page.html | 0 .../templates/users}/manage_account.html | 2 +- .../templates/users/new_role_page.html | 2 +- .../templates/users/request_developer.html | 0 .../{ => users}/templates/users/rolelist.html | 4 +- .../templates/users/user_profile_page.html | 2 +- .../{ => users}/templates/users/userlist.html | 8 +- .../docassemble/webapp/users/views.py | 1103 +- .../docassemble/webapp/{ => utils}/api_key.py | 0 .../docassemble/webapp/utils/constants.py | 4 + .../docassemble/webapp/utils/encryption.py | 109 + .../docassemble/webapp/utils/filenames.py | 69 + .../webapp/{ => utils}/fixpickle.py | 7 +- .../docassemble/webapp/utils/helpers.py | 3538 +++ .../docassemble/webapp/utils/hooks.py | 67 + .../docassemble/webapp/utils/logger.py | 1 + .../docassemble/webapp/utils/path.py | 15 + .../webapp/utils/redis_cred_storage.py | 37 + .../docassemble/webapp/utils/regex.py | 4 + .../docassemble/webapp/utils/request.py | 11 + .../docassemble/webapp/worker.py | 23 +- .../docassemble/webapp/worker_common.py | 167 - .../docassemble/webapp/wsgi_restart.py | 8 +- docassemble_webapp/py.typed | 0 docassemble_webapp/pyproject.toml | 60 +- tests/features/TestExamples.feature | 45 +- tests/features/TestExamplesAdvanced.feature | 21 + tests/features/steps/docassemble.py | 5 +- 405 files changed, 42569 insertions(+), 37292 deletions(-) create mode 100644 docassemble_base/docassemble/base/background.py create mode 100644 docassemble_base/docassemble/base/dates.py create mode 100644 docassemble_base/docassemble/base/empty.py delete mode 100644 docassemble_base/docassemble/base/filter.py rename {docassemble_webapp/docassemble/webapp/templates/pages => docassemble_base/docassemble/base/filter}/__init__.py (100%) create mode 100644 docassemble_base/docassemble/base/filter/docx.py create mode 100644 docassemble_base/docassemble/base/filter/docx_subdoc.py create mode 100644 docassemble_base/docassemble/base/filter/html.py create mode 100644 docassemble_base/docassemble/base/filter/image_docx.py create mode 100644 docassemble_base/docassemble/base/filter/pandoc.py create mode 100644 docassemble_base/docassemble/base/filter/utils.py create mode 100644 docassemble_base/docassemble/base/helpers.py create mode 100644 docassemble_base/docassemble/base/hooks.py create mode 100644 docassemble_base/docassemble/base/hookspecs.py create mode 100644 docassemble_base/docassemble/base/interview_source.py create mode 100644 docassemble_base/docassemble/base/jinja.py rename {docassemble_webapp/docassemble/webapp/templates/users => docassemble_base/docassemble/base/language}/__init__.py (100%) create mode 100644 docassemble_base/docassemble/base/language/capitalization.py create mode 100644 docassemble_base/docassemble/base/language/control.py create mode 100644 docassemble_base/docassemble/base/language/core.py create mode 100644 docassemble_base/docassemble/base/language/currency.py create mode 100644 docassemble_base/docassemble/base/language/language.py create mode 100644 docassemble_base/docassemble/base/language/language_de.py create mode 100644 docassemble_base/docassemble/base/language/language_en.py create mode 100644 docassemble_base/docassemble/base/language/language_es.py create mode 100644 docassemble_base/docassemble/base/language/language_fr.py create mode 100644 docassemble_base/docassemble/base/language/language_it.py create mode 100644 docassemble_base/docassemble/base/language/language_nl.py create mode 100644 docassemble_base/docassemble/base/language/numbers.py create mode 100644 docassemble_base/docassemble/base/language/utils.py create mode 100644 docassemble_base/docassemble/base/language/words.py create mode 100644 docassemble_base/docassemble/base/plugin_manager.py create mode 100644 docassemble_base/docassemble/base/py.typed create mode 100644 docassemble_base/docassemble/base/thread_context.py create mode 100644 docassemble_base/py.typed create mode 100644 docassemble_demo/docassemble/demo/py.typed rename docassemble_demo/docassemble/demo/{random-test.py => random_test.py} (85%) create mode 100644 docassemble_demo/py.typed create mode 100644 docassemble_webapp/docassemble/webapp/admin/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/admin/forms.py create mode 100644 docassemble_webapp/docassemble/webapp/admin/funcs.py rename docassemble_webapp/docassemble/webapp/{templates/pages => admin/templates/admin}/config.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => admin/templates/admin}/interviews.html (87%) rename docassemble_webapp/docassemble/webapp/{templates/pages => admin/templates/admin}/start-embedded.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => admin/templates/admin}/start.html (100%) create mode 100644 docassemble_webapp/docassemble/webapp/admin/views.py create mode 100644 docassemble_webapp/docassemble/webapp/api/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/api/api.py create mode 100644 docassemble_webapp/docassemble/webapp/api/blueprint.py create mode 100644 docassemble_webapp/docassemble/webapp/api/forms.py create mode 100644 docassemble_webapp/docassemble/webapp/api/helpers.py rename docassemble_webapp/docassemble/webapp/{templates/pages => api/templates/api}/manage_api.html (89%) create mode 100644 docassemble_webapp/docassemble/webapp/api/views.py create mode 100644 docassemble_webapp/docassemble/webapp/app_initialize.py delete mode 100644 docassemble_webapp/docassemble/webapp/app_socket.py create mode 100644 docassemble_webapp/docassemble/webapp/auth/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/auth/auth0.py create mode 100644 docassemble_webapp/docassemble/webapp/auth/authentik.py create mode 100644 docassemble_webapp/docassemble/webapp/auth/azure.py create mode 100644 docassemble_webapp/docassemble/webapp/auth/blueprint.py create mode 100644 docassemble_webapp/docassemble/webapp/auth/facebook.py create mode 100644 docassemble_webapp/docassemble/webapp/auth/google.py create mode 100644 docassemble_webapp/docassemble/webapp/auth/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/auth/keycloak.py create mode 100644 docassemble_webapp/docassemble/webapp/auth/miniorange.py create mode 100644 docassemble_webapp/docassemble/webapp/auth/oauth.py create mode 100644 docassemble_webapp/docassemble/webapp/auth/templates/__init__.py rename docassemble_webapp/docassemble/webapp/{templates/flask_user => auth/templates/auth}/google_login.html (85%) create mode 100644 docassemble_webapp/docassemble/webapp/auth/views.py create mode 100644 docassemble_webapp/docassemble/webapp/auth/zitadel.py delete mode 100644 docassemble_webapp/docassemble/webapp/backend.py delete mode 100644 docassemble_webapp/docassemble/webapp/cloud.py create mode 100644 docassemble_webapp/docassemble/webapp/cloud/utils.py create mode 100644 docassemble_webapp/docassemble/webapp/config.py delete mode 100644 docassemble_webapp/docassemble/webapp/core/__init__.py delete mode 100644 docassemble_webapp/docassemble/webapp/core/models.py create mode 100644 docassemble_webapp/docassemble/webapp/cron_tasks/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/cron_tasks/blueprint.py create mode 100644 docassemble_webapp/docassemble/webapp/cron_tasks/cli.py create mode 100644 docassemble_webapp/docassemble/webapp/daglobal/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/daglobal/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/daglobal/hooks.py create mode 100644 docassemble_webapp/docassemble/webapp/daglobal/models.py create mode 100644 docassemble_webapp/docassemble/webapp/db.py create mode 100644 docassemble_webapp/docassemble/webapp/db_base.py delete mode 100644 docassemble_webapp/docassemble/webapp/db_object.py create mode 100644 docassemble_webapp/docassemble/webapp/develop/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/develop/api.py create mode 100644 docassemble_webapp/docassemble/webapp/develop/blueprint.py create mode 100644 docassemble_webapp/docassemble/webapp/develop/common.py rename docassemble_webapp/docassemble/webapp/{develop.py => develop/forms.py} (74%) create mode 100644 docassemble_webapp/docassemble/webapp/develop/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/develop/oauth.py rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/create_package.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/create_playground_package.html (76%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/gd_sync_wait.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/github.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/google_sync.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/googledrive.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/manage_playgrounds.html (84%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/manage_projects.html (66%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/od_sync_wait.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/officeaddin.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/officefunctionfile.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/officemanifest.xml (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/officeouter.html (97%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/onedrive.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/playground.html (68%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/playground_poll.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/playgroundfiles.html (71%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/playgroundpackages.html (87%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/pull_playground_package.html (85%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/rename_project.html (75%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/test_embed.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/update_package_wait.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/utilities.html (88%) rename docassemble_webapp/docassemble/webapp/{templates/pages => develop/templates/develop}/view_source.html (100%) create mode 100644 docassemble_webapp/docassemble/webapp/develop/views.py create mode 100644 docassemble_webapp/docassemble/webapp/emailserver/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/emailserver/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/emailserver/models.py create mode 100644 docassemble_webapp/docassemble/webapp/errors.py create mode 100644 docassemble_webapp/docassemble/webapp/extensions.py create mode 100644 docassemble_webapp/docassemble/webapp/fax/__init__.py rename docassemble_webapp/docassemble/webapp/{ => fax}/clicksend.py (87%) create mode 100644 docassemble_webapp/docassemble/webapp/fax/helpers.py rename docassemble_webapp/docassemble/webapp/{ => fax}/telnyx.py (80%) create mode 100644 docassemble_webapp/docassemble/webapp/fax/views.py delete mode 100644 docassemble_webapp/docassemble/webapp/file_number.py create mode 100644 docassemble_webapp/docassemble/webapp/files/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/files/api.py create mode 100644 docassemble_webapp/docassemble/webapp/files/blueprint.py create mode 100644 docassemble_webapp/docassemble/webapp/files/common.py rename docassemble_webapp/docassemble/webapp/{ => files}/file_access.py (91%) create mode 100644 docassemble_webapp/docassemble/webapp/files/file_number.py create mode 100644 docassemble_webapp/docassemble/webapp/files/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/files/hooks.py rename docassemble_webapp/docassemble/webapp/{ => files}/package_info.py (89%) rename docassemble_webapp/docassemble/webapp/{files.py => files/savedfile.py} (93%) create mode 100644 docassemble_webapp/docassemble/webapp/files/views.py create mode 100755 docassemble_webapp/docassemble/webapp/fixup.sh create mode 100644 docassemble_webapp/docassemble/webapp/flask_app.py create mode 100644 docassemble_webapp/docassemble/webapp/flask_sql_config.py create mode 100644 docassemble_webapp/docassemble/webapp/hooks/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/hooks/default.py create mode 100644 docassemble_webapp/docassemble/webapp/hooks/impl.py create mode 100644 docassemble_webapp/docassemble/webapp/interview/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/interview/api.py create mode 100644 docassemble_webapp/docassemble/webapp/interview/blueprint.py create mode 100644 docassemble_webapp/docassemble/webapp/interview/common.py create mode 100644 docassemble_webapp/docassemble/webapp/interview/config.py create mode 100644 docassemble_webapp/docassemble/webapp/interview/dictionary.py create mode 100644 docassemble_webapp/docassemble/webapp/interview/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/interview/hooks.py create mode 100644 docassemble_webapp/docassemble/webapp/interview/models.py create mode 100644 docassemble_webapp/docassemble/webapp/interview/views.py create mode 100644 docassemble_webapp/docassemble/webapp/jsonstorage/__init__.py rename docassemble_webapp/docassemble/webapp/{jsonstore.py => jsonstorage/helpers.py} (84%) create mode 100644 docassemble_webapp/docassemble/webapp/jsonstorage/models.py create mode 100644 docassemble_webapp/docassemble/webapp/lifecycle.py create mode 100644 docassemble_webapp/docassemble/webapp/lock.py create mode 100644 docassemble_webapp/docassemble/webapp/log_initialize.py create mode 100644 docassemble_webapp/docassemble/webapp/logs/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/logs/forms.py rename docassemble_webapp/docassemble/webapp/{templates/pages => logs/templates/logs}/logs.html (78%) create mode 100644 docassemble_webapp/docassemble/webapp/logs/views.py create mode 100644 docassemble_webapp/docassemble/webapp/mail/__init__.py rename docassemble_webapp/docassemble/webapp/{ => mail}/azure_mail.py (95%) rename docassemble_webapp/docassemble/webapp/{ => mail}/da_flask_mail.py (88%) create mode 100644 docassemble_webapp/docassemble/webapp/mail/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/mail/hooks.py rename docassemble_webapp/docassemble/webapp/{ => mail}/mailgun_mail.py (94%) rename docassemble_webapp/docassemble/webapp/{ => mail}/sendgrid_mail.py (90%) create mode 100644 docassemble_webapp/docassemble/webapp/main/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/main/api.py create mode 100644 docassemble_webapp/docassemble/webapp/main/blueprint.py create mode 100644 docassemble_webapp/docassemble/webapp/main/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/main/hooks.py create mode 100644 docassemble_webapp/docassemble/webapp/main/models.py rename docassemble_webapp/docassemble/webapp/{templates/pages => main/templates/main}/restart.html (100%) create mode 100644 docassemble_webapp/docassemble/webapp/main/views.py create mode 100644 docassemble_webapp/docassemble/webapp/ml/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/ml/blueprint.py create mode 100644 docassemble_webapp/docassemble/webapp/ml/common.py create mode 100644 docassemble_webapp/docassemble/webapp/ml/config.py create mode 100644 docassemble_webapp/docassemble/webapp/ml/forms.py create mode 100644 docassemble_webapp/docassemble/webapp/ml/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/ml/hooks.py rename docassemble_webapp/docassemble/webapp/{ => ml}/machinelearning.py (98%) create mode 100644 docassemble_webapp/docassemble/webapp/ml/models.py rename docassemble_webapp/docassemble/webapp/{templates/pages => ml/templates/ml}/train.html (77%) create mode 100644 docassemble_webapp/docassemble/webapp/ml/views.py create mode 100644 docassemble_webapp/docassemble/webapp/monitor/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/monitor/blueprint.py create mode 100644 docassemble_webapp/docassemble/webapp/monitor/hooks.py create mode 100644 docassemble_webapp/docassemble/webapp/monitor/models.py rename docassemble_webapp/docassemble/webapp/{templates/pages => monitor/templates/monitor}/monitor.html (100%) create mode 100644 docassemble_webapp/docassemble/webapp/monitor/views.py create mode 100644 docassemble_webapp/docassemble/webapp/objectstore/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/objectstore/hooks.py create mode 100644 docassemble_webapp/docassemble/webapp/objectstore/models.py create mode 100644 docassemble_webapp/docassemble/webapp/packages/api.py create mode 100644 docassemble_webapp/docassemble/webapp/packages/blueprint.py create mode 100644 docassemble_webapp/docassemble/webapp/packages/forms.py create mode 100644 docassemble_webapp/docassemble/webapp/packages/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/packages/pip.py rename docassemble_webapp/docassemble/webapp/{templates/pages => packages/templates/packages}/update_package.html (66%) create mode 100644 docassemble_webapp/docassemble/webapp/packages/views.py create mode 100644 docassemble_webapp/docassemble/webapp/phonelogin/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/phonelogin/forms.py rename docassemble_webapp/docassemble/webapp/{templates/flask_user => phonelogin/templates/phonelogin}/phone_login.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/flask_user => phonelogin/templates/phonelogin}/phone_login_verify.html (100%) create mode 100644 docassemble_webapp/docassemble/webapp/phonelogin/views.py create mode 100644 docassemble_webapp/docassemble/webapp/py.typed create mode 100644 docassemble_webapp/docassemble/webapp/react/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/react/api.py rename docassemble_webapp/docassemble/webapp/{ => services}/validators.py (85%) create mode 100644 docassemble_webapp/docassemble/webapp/sessions.py create mode 100644 docassemble_webapp/docassemble/webapp/signals.py create mode 100644 docassemble_webapp/docassemble/webapp/sms/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/sms/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/sms/views.py create mode 100644 docassemble_webapp/docassemble/webapp/tasks/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/tasks/app.py create mode 100644 docassemble_webapp/docassemble/webapp/tasks/app_object.py create mode 100644 docassemble_webapp/docassemble/webapp/tasks/common.py create mode 100644 docassemble_webapp/docassemble/webapp/tasks/context.py create mode 100644 docassemble_webapp/docassemble/webapp/tasks/hooks.py rename docassemble_webapp/docassemble/webapp/{worker_tasks.py => tasks/tasks.py} (69%) delete mode 100644 docassemble_webapp/docassemble/webapp/templates/pages/admin_page.html delete mode 100644 docassemble_webapp/docassemble/webapp/templates/pages/home_page.html delete mode 100644 docassemble_webapp/docassemble/webapp/templates/pages/member_page.html delete mode 100644 docassemble_webapp/docassemble/webapp/templates/pages/packages.html delete mode 100644 docassemble_webapp/docassemble/webapp/templates/pages/socketserver.html delete mode 100644 docassemble_webapp/docassemble/webapp/templates/pages/testgoogledrive.html delete mode 100644 docassemble_webapp/docassemble/webapp/templates/pages/testpost.html delete mode 100644 docassemble_webapp/docassemble/webapp/templates/pages/webrtc.html create mode 100644 docassemble_webapp/docassemble/webapp/testing.py create mode 100644 docassemble_webapp/docassemble/webapp/translation/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/translation/views.py create mode 100644 docassemble_webapp/docassemble/webapp/tts/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/tts/hooks.py create mode 100644 docassemble_webapp/docassemble/webapp/tts/models.py create mode 100644 docassemble_webapp/docassemble/webapp/tts/views.py create mode 100644 docassemble_webapp/docassemble/webapp/twilio/__init__.py create mode 100644 docassemble_webapp/docassemble/webapp/twilio/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/users/api.py create mode 100644 docassemble_webapp/docassemble/webapp/users/blueprint.py create mode 100644 docassemble_webapp/docassemble/webapp/users/common.py create mode 100644 docassemble_webapp/docassemble/webapp/users/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/users/hooks.py create mode 100644 docassemble_webapp/docassemble/webapp/users/loginurl.py create mode 100644 docassemble_webapp/docassemble/webapp/users/signals.py create mode 100644 docassemble_webapp/docassemble/webapp/users/templates/users/__init__.py rename docassemble_webapp/docassemble/webapp/{ => users}/templates/users/add_user_page.html (87%) rename docassemble_webapp/docassemble/webapp/{ => users}/templates/users/edit_user_profile_page.html (100%) rename docassemble_webapp/docassemble/webapp/{templates/pages => users/templates/users}/manage_account.html (86%) rename docassemble_webapp/docassemble/webapp/{ => users}/templates/users/new_role_page.html (81%) rename docassemble_webapp/docassemble/webapp/{ => users}/templates/users/request_developer.html (100%) rename docassemble_webapp/docassemble/webapp/{ => users}/templates/users/rolelist.html (56%) rename docassemble_webapp/docassemble/webapp/{ => users}/templates/users/user_profile_page.html (72%) rename docassemble_webapp/docassemble/webapp/{ => users}/templates/users/userlist.html (89%) rename docassemble_webapp/docassemble/webapp/{ => utils}/api_key.py (100%) create mode 100644 docassemble_webapp/docassemble/webapp/utils/constants.py create mode 100644 docassemble_webapp/docassemble/webapp/utils/encryption.py create mode 100644 docassemble_webapp/docassemble/webapp/utils/filenames.py rename docassemble_webapp/docassemble/webapp/{ => utils}/fixpickle.py (92%) create mode 100644 docassemble_webapp/docassemble/webapp/utils/helpers.py create mode 100644 docassemble_webapp/docassemble/webapp/utils/hooks.py create mode 100644 docassemble_webapp/docassemble/webapp/utils/logger.py create mode 100644 docassemble_webapp/docassemble/webapp/utils/path.py create mode 100644 docassemble_webapp/docassemble/webapp/utils/redis_cred_storage.py create mode 100644 docassemble_webapp/docassemble/webapp/utils/regex.py create mode 100644 docassemble_webapp/docassemble/webapp/utils/request.py delete mode 100644 docassemble_webapp/docassemble/webapp/worker_common.py create mode 100644 docassemble_webapp/py.typed diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 07b7f1123..c6daca6a3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -21,8 +21,6 @@ replace = {new_version}' [bumpversion:file:docassemble_webapp/docassemble/webapp/__init__.py] -[bumpversion:file:docassemble_webapp/docassemble/webapp/setup.py] - [bumpversion:file:Docker/VERSION] [bumpversion:file:docassemble_webapp/docassemble/webapp/data/VERSION.txt] diff --git a/.gitignore b/.gitignore index 8e25131b7..ad549198f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ audit.py conflict_check.yml publish.sh pypi-publish.sh +pypi-publish-dev.sh newversion.sh json_files posttoslack.sh @@ -65,10 +66,16 @@ settings.json .dir-locals.el .flake8 mypy.ini +docassemble_base/mypy.ini +docassemble_demo/mypy.ini +docassemble_webapp/mypy.ini stylelint.config.js .stylelintrc.json .sass-cache/ CLAUDE.md .claude/ .claudeignore -__pycache__ \ No newline at end of file +__pycache__ +pyproject.toml +autoimport.py +runpylint.py \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 24e853fd6..3f75ef40d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Change Log +## [1.10.0] - 2026-06-20 + +### Added + +- The `enable api`, `enable email server`, `enable object storage`, + `enable daglobal`, `enable json storage`, `enable sms interface`, + `enable faxing`, and `enable tts` Configuration directives. +- The `pip trusted host` Configuration directive. + +### Changed + +- The internal organization of the source code has changed + significantly. Documented functionality has not changed, but if your + code imports undocumented names, you may find that the name has + moved to another module. Testing your interviews on a development + server is recommended before upgrading a production server. +- `pluggy` is now used to give `docassemble.base` access to + functionality in `docassemble.webapp`. +- Flask blueprints are now used for features of + `docassemble.webapp`. Configuration directives like `enable + training` and `allow log viewing` will now prevent the importing of + the code underlying the disabled features. Turning off these + features can decrease uWSGI memory usage by 29%. +- SQLAlchemy models now inherit from `sqlalchemy.org.DeclarativeBase` + rather than `db.model`. +- Celery tasks are now invoked using signatures. + ## [1.9.13] - 2026-06-03 ### Fixed diff --git a/Docker/config/config.yml.dist b/Docker/config/config.yml.dist index ac41fa1a1..1386874c8 100644 --- a/Docker/config/config.yml.dist +++ b/Docker/config/config.yml.dist @@ -149,3 +149,13 @@ gotenberg: password: {{GOTENBERGPASSWORD}} javascript defer: True use nginx to serve files: {{USENGINXTOSERVEFILES}} +enable api: True +enable daglobal: True +enable email server: True +enable faxing: True +enable json storage: True +enable monitor: True +enable object storage: True +enable sms interface: True +enable training: True +enable tts: True diff --git a/Docker/cron/docassemble-cron-daily.sh b/Docker/cron/docassemble-cron-daily.sh index 9c598969a..3fb6a53d3 100755 --- a/Docker/cron/docassemble-cron-daily.sh +++ b/Docker/cron/docassemble-cron-daily.sh @@ -42,15 +42,17 @@ if [[ $CONTAINERROLE =~ .*:(all):.* ]] && [ "${USEHTTPS:-false}" == "true" ] && cp /tmp/letsencrypt.tar.gz "${DA_ROOT}/backup/letsencrypt.tar.gz" fi rm -f /tmp/letsencrypt.tar.gz - if [ ! -f /etc/ssl/docassemble/exim.crt ] && [ ! -f /etc/ssl/docassemble/exim.key ]; then - cp "/etc/letsencrypt/live/${DAHOSTNAME}/fullchain.pem" /etc/exim4/exim.crt - cp "/etc/letsencrypt/live/${DAHOSTNAME}/privkey.pem" /etc/exim4/exim.key - chown root:Debian-exim /etc/exim4/exim.crt - chown root:Debian-exim /etc/exim4/exim.key - chmod 640 /etc/exim4/exim.crt - chmod 640 /etc/exim4/exim.key - supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 stop exim4 - supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 start exim4 + if [ "$ENABLEEMAILSERVER" == "true" ]; then + if [ ! -f /etc/ssl/docassemble/exim.crt ] && [ ! -f /etc/ssl/docassemble/exim.key ]; then + cp "/etc/letsencrypt/live/${DAHOSTNAME}/fullchain.pem" /etc/exim4/exim.crt + cp "/etc/letsencrypt/live/${DAHOSTNAME}/privkey.pem" /etc/exim4/exim.key + chown root:Debian-exim /etc/exim4/exim.crt + chown root:Debian-exim /etc/exim4/exim.key + chmod 640 /etc/exim4/exim.crt + chmod 640 /etc/exim4/exim.key + supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 stop exim4 + supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 start exim4 + fi fi fi diff --git a/Docker/initialize.sh b/Docker/initialize.sh index 49d170fad..04042a7a5 100755 --- a/Docker/initialize.sh +++ b/Docker/initialize.sh @@ -1339,8 +1339,10 @@ if [ "${DAWEBSERVER:-nginx}" = "none" ]; then ${SUPERVISORCMD} stop nascent &> /dev/null NASCENTRUNNING=false; if [[ $CONTAINERROLE =~ .*:(all|web):.* ]]; then - echo "initialize: Starting websockets" >&2 - ${SUPERVISORCMD} start websockets + if [ "${ENABLEMONITOR:-true}" = "true" ]; then + echo "initialize: Starting websockets" >&2 + ${SUPERVISORCMD} start websockets + fi echo "initialize: Starting uwsgi" >&2 ${SUPERVISORCMD} start uwsgi fi @@ -1427,8 +1429,10 @@ if [ "${DAWEBSERVER:-nginx}" = "nginx" ]; then fi fi if [[ $CONTAINERROLE =~ .*:(all|web):.* ]]; then - echo "initialize: Starting websockets" >&2 - ${SUPERVISORCMD} start websockets + if [ "${ENABLEMONITOR:-true}" = "true" ]; then + echo "initialize: Starting websockets" >&2 + ${SUPERVISORCMD} start websockets + fi echo "initialize: Starting uwsgi" >&2 ${SUPERVISORCMD} start uwsgi fi @@ -1575,7 +1579,7 @@ if [ "${DAWEBSERVER:-nginx}" = "apache" ]; then a2ensite docassemble-log fi - if [[ $CONTAINERROLE =~ .*:(all|web):.* ]]; then + if [[ $CONTAINERROLE =~ .*:(all|web):.* ]] && [ "${ENABLEMONITOR:-true}" = "true" ]; then echo "initialize: Starting websockets" >&2 ${SUPERVISORCMD} start websockets fi @@ -1628,81 +1632,83 @@ if [ "$CRONRUNNING" == "false" ]; then ${SUPERVISORCMD} start cron fi -if exiwhat 2> /dev/null | grep -q listening; then - EXIM4RUNNING=true -else - EXIM4RUNNING=false -fi +if [ "${ENABLEEMAILSERVER:-true}" == "true" ]; then + if exiwhat 2> /dev/null | grep -q listening; then + EXIM4RUNNING=true + else + EXIM4RUNNING=false + fi -if [ "$EXIM4RUNNING" == "false" ] && [[ $CONTAINERROLE =~ .*:(all|mail):.* && ($DBTYPE = "postgresql" || $DBTYPE = "mysql") ]]; then - echo "initialize: Starting exim4" >&2 - if [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then - if [ -f /usr/share/docassemble/config/exim4-update ] && [ "${DAHOSTNAME}" != "localhost" ]; then - sed "s/dc_other_hostnames='\**'/dc_other_hostnames='${DAHOSTNAME}'/" /usr/share/docassemble/config/exim4-update > /tmp/temp-exim4-update - if [ ! -f /etc/exim4/update-exim4.conf.conf ] || ! cmp -s /tmp/temp-exim4-update /etc/exim4/update-exim4.conf.conf; then - cp /tmp/temp-exim4-update /etc/exim4/update-exim4.conf.conf - update-exim4.conf - fi - rm -f /tmp/temp-exim4-update - fi - rm -f /etc/cron.daily/exim4-base - ln -s /usr/share/docassemble/cron/exim4-base /etc/cron.daily/exim4-base - if [ "${DBTYPE}" = "postgresql" ]; then - cp "${DA_ROOT}/config/exim4-router-postgresql" /etc/exim4/dbrouter - if [ "${DBHOST:-null}" != "null" ]; then - echo -n 'hide pgsql_servers = '${DBHOST} > /etc/exim4/dbinfo - else - echo -n 'hide pgsql_servers = localhost' > /etc/exim4/dbinfo - fi - if [ "${DBPORT:-null}" != "null" ]; then - echo -n '::'${DBPORT} >> /etc/exim4/dbinfo - fi - echo '/'${DBNAME}'/'${DBUSER}'/'${DBPASSWORD} >> /etc/exim4/dbinfo - fi - if [ "$DBTYPE" = "mysql" ]; then - cp "${DA_ROOT}/config/exim4-router-mysql" /etc/exim4/dbrouter - if [ "${DBHOST:-null}" != "null" ]; then - echo -n 'hide mysql_servers = '${DBHOST} > /etc/exim4/dbinfo - else - echo -n 'hide mysql_servers = localhost' > /etc/exim4/dbinfo - fi - if [ "${DBPORT:-null}" != "null" ]; then - echo -n '::'${DBPORT} >> /etc/exim4/dbinfo - fi - echo '/'${DBNAME}'/'${DBUSER}'/'${DBPASSWORD} >> /etc/exim4/dbinfo - fi - if [ "${DBTYPE}" = "postgresql" ]; then - echo 'DAQUERY = select short from '${DBTABLEPREFIX}"shortener where short='\${quote_pgsql:\$local_part}'" >> /etc/exim4/dbinfo - fi - if [ "${DBTYPE}" = "mysql" ]; then - echo 'DAQUERY = select short from '${DBTABLEPREFIX}"shortener where short='\${quote_mysql:\$local_part}'" >> /etc/exim4/dbinfo - fi - if [ -f /etc/ssl/docassemble/exim.crt ] && [ -f /etc/ssl/docassemble/exim.key ]; then - cp /etc/ssl/docassemble/exim.crt /etc/exim4/exim.crt - cp /etc/ssl/docassemble/exim.key /etc/exim4/exim.key - chown root:Debian-exim /etc/exim4/exim.crt - chown root:Debian-exim /etc/exim4/exim.key - chmod 640 /etc/exim4/exim.crt - chmod 640 /etc/exim4/exim.key - echo 'MAIN_TLS_ENABLE = yes' >> /etc/exim4/dbinfo - elif [[ $CONTAINERROLE =~ .*:(all|web):.* ]] && [ "${USELETSENCRYPT:-false}" == "true" ] && [ -f "/etc/letsencrypt/live/${DAHOSTNAME}/cert.pem" ] && [ -f "/etc/letsencrypt/live/${DAHOSTNAME}/privkey.pem" ]; then - cp "/etc/letsencrypt/live/${DAHOSTNAME}/fullchain.pem" /etc/exim4/exim.crt - cp "/etc/letsencrypt/live/${DAHOSTNAME}/privkey.pem" /etc/exim4/exim.key - chown root:Debian-exim /etc/exim4/exim.crt - chown root:Debian-exim /etc/exim4/exim.key - chmod 640 /etc/exim4/exim.crt - chmod 640 /etc/exim4/exim.key - echo 'MAIN_TLS_ENABLE = yes' >> /etc/exim4/dbinfo - else - echo 'MAIN_TLS_ENABLE = no' >> /etc/exim4/dbinfo - fi - chmod og-rwx /etc/exim4/dbinfo + if [ "$EXIM4RUNNING" == "false" ] && [[ $CONTAINERROLE =~ .*:(all|mail):.* && ($DBTYPE = "postgresql" || $DBTYPE = "mysql") ]]; then + echo "initialize: Starting exim4" >&2 + if [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then + if [ -f /usr/share/docassemble/config/exim4-update ] && [ "${DAHOSTNAME}" != "localhost" ]; then + sed "s/dc_other_hostnames='\**'/dc_other_hostnames='${DAHOSTNAME}'/" /usr/share/docassemble/config/exim4-update > /tmp/temp-exim4-update + if [ ! -f /etc/exim4/update-exim4.conf.conf ] || ! cmp -s /tmp/temp-exim4-update /etc/exim4/update-exim4.conf.conf; then + cp /tmp/temp-exim4-update /etc/exim4/update-exim4.conf.conf + update-exim4.conf + fi + rm -f /tmp/temp-exim4-update + fi + rm -f /etc/cron.daily/exim4-base + ln -s /usr/share/docassemble/cron/exim4-base /etc/cron.daily/exim4-base + if [ "${DBTYPE}" = "postgresql" ]; then + cp "${DA_ROOT}/config/exim4-router-postgresql" /etc/exim4/dbrouter + if [ "${DBHOST:-null}" != "null" ]; then + echo -n 'hide pgsql_servers = '${DBHOST} > /etc/exim4/dbinfo + else + echo -n 'hide pgsql_servers = localhost' > /etc/exim4/dbinfo + fi + if [ "${DBPORT:-null}" != "null" ]; then + echo -n '::'${DBPORT} >> /etc/exim4/dbinfo + fi + echo '/'${DBNAME}'/'${DBUSER}'/'${DBPASSWORD} >> /etc/exim4/dbinfo + fi + if [ "$DBTYPE" = "mysql" ]; then + cp "${DA_ROOT}/config/exim4-router-mysql" /etc/exim4/dbrouter + if [ "${DBHOST:-null}" != "null" ]; then + echo -n 'hide mysql_servers = '${DBHOST} > /etc/exim4/dbinfo + else + echo -n 'hide mysql_servers = localhost' > /etc/exim4/dbinfo + fi + if [ "${DBPORT:-null}" != "null" ]; then + echo -n '::'${DBPORT} >> /etc/exim4/dbinfo + fi + echo '/'${DBNAME}'/'${DBUSER}'/'${DBPASSWORD} >> /etc/exim4/dbinfo + fi + if [ "${DBTYPE}" = "postgresql" ]; then + echo 'DAQUERY = select short from '${DBTABLEPREFIX}"shortener where short='\${quote_pgsql:\$local_part}'" >> /etc/exim4/dbinfo + fi + if [ "${DBTYPE}" = "mysql" ]; then + echo 'DAQUERY = select short from '${DBTABLEPREFIX}"shortener where short='\${quote_mysql:\$local_part}'" >> /etc/exim4/dbinfo + fi + if [ -f /etc/ssl/docassemble/exim.crt ] && [ -f /etc/ssl/docassemble/exim.key ]; then + cp /etc/ssl/docassemble/exim.crt /etc/exim4/exim.crt + cp /etc/ssl/docassemble/exim.key /etc/exim4/exim.key + chown root:Debian-exim /etc/exim4/exim.crt + chown root:Debian-exim /etc/exim4/exim.key + chmod 640 /etc/exim4/exim.crt + chmod 640 /etc/exim4/exim.key + echo 'MAIN_TLS_ENABLE = yes' >> /etc/exim4/dbinfo + elif [[ $CONTAINERROLE =~ .*:(all|web):.* ]] && [ "${USELETSENCRYPT:-false}" == "true" ] && [ -f "/etc/letsencrypt/live/${DAHOSTNAME}/cert.pem" ] && [ -f "/etc/letsencrypt/live/${DAHOSTNAME}/privkey.pem" ]; then + cp "/etc/letsencrypt/live/${DAHOSTNAME}/fullchain.pem" /etc/exim4/exim.crt + cp "/etc/letsencrypt/live/${DAHOSTNAME}/privkey.pem" /etc/exim4/exim.key + chown root:Debian-exim /etc/exim4/exim.crt + chown root:Debian-exim /etc/exim4/exim.key + chmod 640 /etc/exim4/exim.crt + chmod 640 /etc/exim4/exim.key + echo 'MAIN_TLS_ENABLE = yes' >> /etc/exim4/dbinfo + else + echo 'MAIN_TLS_ENABLE = no' >> /etc/exim4/dbinfo + fi + chmod og-rwx /etc/exim4/dbinfo + fi + ${SUPERVISORCMD} start exim4 + elif [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then + echo "initialize: Disabling exim4 cron" >&2 + rm -f /etc/cron.daily/exim4-base + ln -s /usr/share/docassemble/cron/donothing /etc/cron.daily/exim4-base fi - ${SUPERVISORCMD} start exim4 -elif [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then - echo "initialize: Disabling exim4 cron" >&2 - rm -f /etc/cron.daily/exim4-base - ln -s /usr/share/docassemble/cron/donothing /etc/cron.daily/exim4-base fi if [[ $CONTAINERROLE =~ .*:(log):.* ]] || [ "$OTHERLOGSERVER" == "true" ]; then diff --git a/Docker/reset.sh b/Docker/reset.sh index 23371e7ca..20bce3f69 100755 --- a/Docker/reset.sh +++ b/Docker/reset.sh @@ -75,6 +75,12 @@ update_pip_config(){ else pip config unset global.extra-index-url &> /dev/null fi + + if [ "${PIPTRUSTEDHOST:-null}" != "null" ]; then + pip config set global.trusted-host "${PIPTRUSTEDHOST}" + else + pip config unset global.trusted-host &> /dev/null + fi } restart_websockets() { echo "`date` stopping websockets" >&2 @@ -108,7 +114,7 @@ if [ "${DAALLOWUPDATES:-true}" == "true" ]; then update_pip_config & fi -if [[ $CONTAINERROLE =~ .*:(all|web):.* ]]; then +if [[ $CONTAINERROLE =~ .*:(all|web):.* ]] && [ "${ENABLEMONITOR:-true}" = "true" ]; then restart_websockets & fi diff --git a/Docker/restart-post-logrotate.sh b/Docker/restart-post-logrotate.sh index 525e88b91..a478a74c9 100755 --- a/Docker/restart-post-logrotate.sh +++ b/Docker/restart-post-logrotate.sh @@ -39,7 +39,7 @@ if [[ $CONTAINERROLE =~ .*:(all|celery):.* ]]; then ${SUPERVISORCMD} start celerysingle > /dev/null || exit 1 fi -if [[ $CONTAINERROLE =~ .*:(all|web):.* ]]; then +if [[ $CONTAINERROLE =~ .*:(all|web):.* ]] && [ "${ENABLEMONITOR:-true}" = "true" ]; then ${SUPERVISORCMD} stop websockets > /dev/null || exit 1 sleep 1 ${SUPERVISORCMD} start websockets > /dev/null || exit 1 diff --git a/Docker/run-apache.sh b/Docker/run-apache.sh index a652aca24..861f39b28 100755 --- a/Docker/run-apache.sh +++ b/Docker/run-apache.sh @@ -12,7 +12,7 @@ source /dev/stdin < <(su -c "source \"$DA_ACTIVATE\" && python -m docassemble.ba set -- $LOCALE export LANG=$1 -if [[ '$(dpkg --print-architecture)' == 'amd64' ]]; then +if [[ $(dpkg --print-architecture) == 'amd64' ]]; then CURRENTARCH=x86_64 else CURRENTARCH=aarch64 diff --git a/Docker/run-cron.sh b/Docker/run-cron.sh index a8f4e74ca..ffa734456 100755 --- a/Docker/run-cron.sh +++ b/Docker/run-cron.sh @@ -16,4 +16,6 @@ source /dev/stdin < <(su -c "source \"$DA_ACTIVATE\" && python -m docassemble.ba set -- $LOCALE export LANG=$1 -exec nice -n 19 su -c "source \"$DA_ACTIVATE\" && python -m docassemble.webapp.cron \"$DA_CONFIG_FILE\" -type $CRONTYPE" www-data +export IN_CRON=true + +exec nice -n 19 su -c "source \"$DA_ACTIVATE\" && flask --app docassemble.webapp.server cron run $CRONTYPE" www-data diff --git a/Dockerfile b/Dockerfile index 3143b6ae6..25acc97d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,9 +54,9 @@ bash -c \ && pip install --upgrade pip==26.0.1 \ && pip install --upgrade mod_wsgi==5.0.2 \ && pip install --upgrade \ - certbot==5.2.2 \ - certbot-apache==5.2.2 \ - certbot-nginx==5.2.2 \ + certbot==5.6.0 \ + certbot-apache==5.6.0 \ + certbot-nginx==5.6.0 \ minio==7.2.20 \ uWSGI==2.0.31 \ && pip install \ diff --git a/LICENSE.txt b/LICENSE.txt index f9a98fdcf..a3c07394e 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015-2025 Jonathan Pyle +Copyright (c) 2015-2026 Jonathan Pyle Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docassemble_base/LICENSE.txt b/docassemble_base/LICENSE.txt index f9a98fdcf..a3c07394e 100644 --- a/docassemble_base/LICENSE.txt +++ b/docassemble_base/LICENSE.txt @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015-2025 Jonathan Pyle +Copyright (c) 2015-2026 Jonathan Pyle Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docassemble_base/config.yml b/docassemble_base/config.yml index c931d83c6..398af6b3c 100644 --- a/docassemble_base/config.yml +++ b/docassemble_base/config.yml @@ -99,3 +99,13 @@ allow configuration editing: True javascript defer: True use nginx to serve files: True restrict input variables: True +enable api: True +enable daglobal: True +enable email server: True +enable faxing: True +enable json storage: True +enable monitor: True +enable object storage: True +enable sms interface: True +enable training: True +enable tts: True diff --git a/docassemble_base/docassemble/base/DA.py b/docassemble_base/docassemble/base/DA.py index 8ee28270c..dadd365e7 100644 --- a/docassemble_base/docassemble/base/DA.py +++ b/docassemble_base/docassemble/base/DA.py @@ -1,3 +1,4 @@ +# pylint: disable=invalid-name __all__ = [] diff --git a/docassemble_base/docassemble/base/amazon.py b/docassemble_base/docassemble/base/amazon.py index c04c0f54b..a796dd44b 100644 --- a/docassemble_base/docassemble/base/amazon.py +++ b/docassemble_base/docassemble/base/amazon.py @@ -7,7 +7,7 @@ epoch = datetime.datetime(1970, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) -class s3object: +class S3Object: def __init__(self, s3_config): self.upload_args = {} @@ -41,23 +41,23 @@ def __init__(self, s3_config): self.bucket_name = s3_config['bucket'] def get_key(self, key_name): - return s3key(self, self.conn.Object(self.bucket_name, key_name)) + return S3Key(self, self.conn.Object(self.bucket_name, key_name)) def search_key(self, key_name): for key in self.bucket.objects.filter(Prefix=key_name, Delimiter='/'): if key.key == key_name: - return s3key(self, self.conn.Object(self.bucket_name, key.key)) + return S3Key(self, self.conn.Object(self.bucket_name, key.key)) return None def list_keys(self, prefix): output = [] for obj in self.bucket.objects.filter(Prefix=prefix): - new_key = s3key(self, obj) + new_key = S3Key(self, obj) output.append(new_key) return output -class s3key: +class S3Key: def __init__(self, s3_object, key_obj): self.s3_object = s3_object diff --git a/docassemble_base/docassemble/base/astparser.py b/docassemble_base/docassemble/base/astparser.py index 02dafcbca..5e1f7f599 100644 --- a/docassemble_base/docassemble/base/astparser.py +++ b/docassemble_base/docassemble/base/astparser.py @@ -1,3 +1,4 @@ +# pylint: disable=invalid-name import ast import re @@ -7,7 +8,7 @@ valid_variable_match = re.compile(r'^[^\d][A-Za-z0-9\_]*$') -class myextract(ast.NodeVisitor): +class MyExtract(ast.NodeVisitor): def __init__(self): self.stack = [] @@ -66,7 +67,7 @@ def visit_Subscript(self, node): self.in_subscript -= 1 -class myvisitnode(ast.NodeVisitor): +class MyVisitNode(ast.NodeVisitor): def __init__(self): self.names = {} @@ -90,7 +91,7 @@ def visit_Call(self, node): def visit_Subscript(self, node): if node not in self.calls: - crawler = myextract() + crawler = MyExtract() crawler.visit(node) if not crawler.seen_complexity: self.names[fix_assign.sub(r'\1', (".".join(reversed(crawler.stack))))] = 1 @@ -98,7 +99,7 @@ def visit_Subscript(self, node): def visit_Attribute(self, node): if node not in self.calls: - crawler = myextract() + crawler = MyExtract() crawler.visit(node) if not crawler.seen_complexity: self.names[fix_assign.sub(r'\1', (".".join(reversed(crawler.stack))))] = 1 @@ -117,11 +118,11 @@ def visit_Assign(self, node): for subnode in val: if isinstance(subnode, ast.Tuple): for subsubnode in subnode.elts: - crawler = myextract() + crawler = MyExtract() crawler.visit(subsubnode) self.targets[fix_assign.sub(r'\1', ".".join(reversed(crawler.stack)))] = 1 else: - crawler = myextract() + crawler = MyExtract() crawler.visit(subnode) self.targets[fix_assign.sub(r'\1', ".".join(reversed(crawler.stack)))] = 1 self.depth += 1 @@ -132,7 +133,7 @@ def visit_Assign(self, node): def visit_AugAssign(self, node): for key, val in ast.iter_fields(node): if key == 'target': - crawler = myextract() + crawler = MyExtract() crawler.visit(val) self.targets[fix_assign.sub(r'\1', ".".join(reversed(crawler.stack)))] = 1 self.depth += 1 @@ -143,7 +144,7 @@ def visit_AugAssign(self, node): def visit_AnnAssign(self, node): for key, val in ast.iter_fields(node): if key == 'target': - crawler = myextract() + crawler = MyExtract() crawler.visit(val) self.targets[fix_assign.sub(r'\1', ".".join(reversed(crawler.stack)))] = 1 self.depth += 1 @@ -241,7 +242,7 @@ def visit_Name(self, node): self.generic_visit(node) -class detectIllegal(ast.NodeVisitor): +class DetectIllegal(ast.NodeVisitor): def __init__(self): self.illegal = False @@ -467,7 +468,7 @@ def visit_Starred(self, node): ast.NodeVisitor.generic_visit(self, node) -class detectIllegalQuery(ast.NodeVisitor): +class DetectIllegalQuery(ast.NodeVisitor): def __init__(self): self.illegal = False diff --git a/docassemble_base/docassemble/base/background.py b/docassemble_base/docassemble/base/background.py new file mode 100644 index 000000000..824724ab2 --- /dev/null +++ b/docassemble_base/docassemble/base/background.py @@ -0,0 +1,63 @@ +from docassemble.base.thread_context import this_thread +from docassemble.base.hooks import get_server_redis, get_celery_app, get_task + + +class BackgroundResult: + + def __init__(self, result): + for attr in ('value', 'error_type', 'error_trace', 'error_message', 'variables'): + if hasattr(result, attr): + setattr(self, attr, getattr(result, attr)) + else: + setattr(self, attr, None) + + +class MyAsyncResult: + + def wait(self): + if not hasattr(self, '_cached_result'): + self._cached_result = BackgroundResult(get_task(self.obj).get()) + return True + + def failed(self): + if not hasattr(self, '_cached_result'): + self._cached_result = BackgroundResult(get_task(self.obj).get()) + if self._cached_result.error_type is not None: + return True + return False + + def ready(self): + return get_task(self.obj).ready() + + def result(self): + if not hasattr(self, '_cached_result'): + self._cached_result = BackgroundResult(get_task(self.obj).get()) + return self._cached_result + + def get(self): + if not hasattr(self, '_cached_result'): + self._cached_result = BackgroundResult(get_task(self.obj).get()) + return self._cached_result.value + + def revoke(self, terminate=True): + return get_task(self.obj).revoke(terminate=terminate) + + def status(self): + return get_task(self.obj).status + + def state(self): + return get_task(self.obj).state + + def date_done(self): + return get_task(self.obj).date_done + + +def bg_action(action, ui_notification, **kwargs): + result = MyAsyncResult() + result.obj = get_celery_app().signature('tasks.background_action', args=[this_thread.current_info['yaml_filename'], this_thread.current_info['user'], this_thread.current_info['session'], this_thread.current_info['secret'], this_thread.current_info['url'], this_thread.current_info['url_root'], {'action': action, 'arguments': kwargs}], kwargs={"extra": ui_notification}).delay() + if ui_notification is not None: + worker_key = 'da:worker:uid:' + str(this_thread.current_info['session']) + ':i:' + str(this_thread.current_info['yaml_filename']) + ':userid:' + str(this_thread.current_info['user']['the_user_id']) + # logmessage("worker_caller: id is " + str(result.obj.id) + " and key is " + worker_key) + get_server_redis().rpush(worker_key, result.obj.id) + # logmessage("worker_caller: id is " + str(result.obj.id)) + return result diff --git a/docassemble_base/docassemble/base/config.py b/docassemble_base/docassemble/base/config.py index dc09df996..5b949e322 100644 --- a/docassemble_base/docassemble/base/config.py +++ b/docassemble_base/docassemble/base/config.py @@ -1,3 +1,4 @@ +# mypy: disable-error-code="attr-defined" import os import re import sys @@ -12,8 +13,10 @@ import httplib2 from docassemble.base.generate_key import random_string +re._MAXCACHE = 10000 # pylint: disable=protected-access +re._MAXCACHE2 = 5000 # pylint: disable=protected-access START_TIME = time.time() -dbtableprefix = None +dbtableprefix = None # pylint: disable=invalid-name daconfig = {} s3_config = {} S3_ENABLED = False @@ -21,10 +24,10 @@ GC_ENABLED = False azure_config = {} AZURE_ENABLED = False -hostname = None -loaded = False -in_celery = False -in_cron = False +hostname = None # pylint: disable=invalid-name +loaded = False # pylint: disable=invalid-name +in_celery = False # pylint: disable=invalid-name +in_cron = False # pylint: disable=invalid-name errors = [] env_messages = [] allowed = {} @@ -274,7 +277,7 @@ def load(**kwargs): filename = kwargs.get('filename', os.getenv('DA_CONFIG_FILE', '/usr/share/docassemble/config/config.yml')) if 'in_celery' in kwargs and kwargs['in_celery']: in_celery = True - if 'in_cron' in kwargs and kwargs['in_cron']: + if ('in_cron' in kwargs and kwargs['in_cron']) or env_translate('IN_CRON'): in_cron = True if not os.path.isfile(filename): if not os.access(os.path.dirname(filename), os.W_OK): @@ -515,10 +518,10 @@ def load(**kwargs): hostname = os.getenv('SERVERHOSTNAME', socket.gethostname()) if S3_ENABLED: import docassemble.base.amazon # pylint: disable=import-outside-toplevel - cloud = docassemble.base.amazon.s3object(s3_config) + cloud = docassemble.base.amazon.S3Object(s3_config) elif AZURE_ENABLED: import docassemble.base.microsoft # pylint: disable=import-outside-toplevel - cloud = docassemble.base.microsoft.azureobject(azure_config) + cloud = docassemble.base.microsoft.AzureObject(azure_config) if ('key vault name' in azure_config and azure_config['key vault name'] is not None and 'managed identity' in azure_config and azure_config['managed identity'] is not None): daconfig = cloud.load_with_secrets(daconfig) else: @@ -975,6 +978,12 @@ def load(**kwargs): if new_filename: new_delete_days[new_filename] = days daconfig['interview delete days by filename'] = new_delete_days + if 'celery result retention days' in daconfig: + if not isinstance(daconfig['celery result retention days'], (int, float)) and daconfig['celery result retention days'] >= 0: + config_error('celery result retention days must be an int or a float greater than or equal to zero') + daconfig['celery result retention days'] = 1 + else: + daconfig['celery result retention days'] = 1 for key in ('default interview', 'session list interview', 'dispatch interview', 'auto resume interview'): if key in daconfig: if isinstance(daconfig[key], str): diff --git a/docassemble_base/docassemble/base/core.py b/docassemble_base/docassemble/base/core.py index 8d71f24cc..5c07630d5 100644 --- a/docassemble_base/docassemble/base/core.py +++ b/docassemble_base/docassemble/base/core.py @@ -1,6 +1,36 @@ +# ruff: noqa: F401 +# pylint: disable=unused-import # This module imports names for backwards compatibility and to ensure # that pickled objects in existing sessions can be unpickled. __all__ = ['DAObject', 'DAList', 'DADict', 'DAOrderedDict', 'DASet', 'DAFile', 'DAFileCollection', 'DAFileList', 'DAStaticFile', 'DAEmail', 'DAEmailRecipient', 'DAEmailRecipientList', 'DATemplate', 'DAEmpty', 'DALink', 'RelationshipTree', 'DAContext'] -from docassemble.base.util import DAObject, DAList, DADict, DAOrderedDict, DASet, DAFile, DAFileCollection, DAFileList, DAStaticFile, DAEmail, DAEmailRecipient, DAEmailRecipientList, DATemplate, DAEmpty, DALink, RelationshipTree, DAContext, DAObjectPlusParameters, DACatchAll, RelationshipDir, RelationshipPeer, DALazyTemplate, DALazyTableTemplate, selections, DASessionLocal, DADeviceLocal, DAUserLocal # noqa: F401 # pylint: disable=unused-import +from docassemble.base.util import ( + DAObject, + DAList, + DADict, + DAOrderedDict, + DASet, + DAFile, + DAFileCollection, + DAFileList, + DAStaticFile, + DAEmail, + DAEmailRecipient, + DAEmailRecipientList, + DATemplate, + DAEmpty, + DALink, + RelationshipTree, + DAContext, + DAObjectPlusParameters, + DACatchAll, + RelationshipDir, + RelationshipPeer, + DALazyTemplate, + DALazyTableTemplate, + selections, + DASessionLocal, + DADeviceLocal, + DAUserLocal, +) diff --git a/docassemble_base/docassemble/base/data/sources/base-words.yml b/docassemble_base/docassemble/base/data/sources/base-words.yml index 8d7b7a5de..1bf121478 100644 --- a/docassemble_base/docassemble/base/data/sources/base-words.yml +++ b/docassemble_base/docassemble/base/data/sources/base-words.yml @@ -55,7 +55,6 @@ "API Keys": Null "Apply": Null "App": Null -", are attached": Null "A reset password email has been sent to '%(email)s'. Open that email and follow the instructions to reset your password.": Null "are we": Null "are you": Null @@ -381,7 +380,6 @@ "Google Drive": Null "Google Drive synchronization": Null "Google Drive Sync": Null -"Google Drive Test": Null "Greece": Null "Greenland": Null "Grenada": Null @@ -500,7 +498,6 @@ "Iraq": Null "Ireland": Null ", is already in use by someone else": Null -", is attached": Null "Isle of Man": Null "is not a valid date and time.": Null "is not a valid date.": Null @@ -701,6 +698,7 @@ "Other settings": Null "other user": Null "other users": Null +"our": Null "ourselves": Null "Output Format": Null "Overrides": Null @@ -1004,7 +1002,6 @@ "(term definition)": Null "Terms used in this question:": Null "Terms used:": Null -"Test": Null "Thailand": Null "Thank you for registering": Null "Thank you for registering with": Null @@ -1330,10 +1327,8 @@ "Your confirmation token has expired.": Null "Your docassemble server": Null "Your document is attached.": Null -"Your document, ": Null "Your documents are attached.": Null "Your document, %s, is attached.": Null -"Your documents, ": Null "Your documents, %s, are attached.": Null "Your documents will be e-mailed to": Null 'Your email address has not yet been confirmed. Check your email Inbox and Spam folders for the confirmation email or Re-send confirmation email.': Null diff --git a/docassemble_base/docassemble/base/dates.py b/docassemble_base/docassemble/base/dates.py new file mode 100644 index 000000000..7a7fe14ec --- /dev/null +++ b/docassemble_base/docassemble/base/dates.py @@ -0,0 +1,484 @@ +try: + import zoneinfo +except ImportError: + from backports import zoneinfo # type: ignore[no-redef] +import datetime +import dateutil +import dateutil.parser +import babel.dates +from .empty import DAEmpty +from .hooks import get_configuration, get_default_timezone +from .language.capitalization import capitalize +from .language.control import get_language +from .language.core import ensure_definition +from .language.language import noun_plural, comma_and_list +from .language.numbers import nice_number +from .language.words import word +from .thread_context import this_thread + +def interview_default(the_part, default_value, language): + if the_part in this_thread.internal and this_thread.internal[the_part] is not None: + return this_thread.internal[the_part] + for lang in (language, get_language(), '*'): + if lang is not None and this_thread.interview is not None and lang in this_thread.interview.default_title and the_part in this_thread.interview.default_title[lang]: + return this_thread.interview.default_title[lang][the_part] + return default_value + +def today(timezone=None, format=None): # pylint: disable=redefined-builtin + """Return today's date at midnight as a DADateTime object. + + Args: + timezone (str or None): IANA timezone name. If None, the interview's + default timezone is used. + format (str or None): If provided, return the date formatted as a + string using this Babel date-format pattern instead of a + DADateTime. + + Returns: + DADateTime or str: Midnight today in the given timezone, or a + formatted date string if ``format`` is specified. + """ + ensure_definition(timezone, format) + if timezone is None: + timezone = get_default_timezone() + val = datetime.datetime.now(datetime.timezone.utc).astimezone(zoneinfo.ZoneInfo(timezone)) + if format is not None: + return dd(val.replace(hour=0, minute=0, second=0, microsecond=0)).format_date(format) + return dd(val.replace(hour=0, minute=0, second=0, microsecond=0)) + + +def babel_language(language): + if 'babel dates map' not in get_configuration(): + return language + return get_configuration()['babel dates map'].get(language, language) + + +def month_of(the_date, as_word=False, language=None): + """Return the month component of a date. + + Args: + the_date (datetime.date, datetime.datetime, or str): The date to + extract the month from. + as_word (bool): If True, return the full month name (e.g. + ``'January'``); otherwise return the month as an integer. + language (str or None): Language code for localizing the month name. + Defaults to the current interview language. + + Returns: + int or str: Month number (1ā€“12) or localized month name. + """ + ensure_definition(the_date, as_word, language) + if language is None: + language = get_language() + try: + if isinstance(the_date, (datetime.datetime, datetime.date)): + date = the_date + else: + date = dateutil.parser.parse(the_date) + if as_word: + return babel.dates.format_date(date, format='MMMM', locale=babel_language(language)) + return int(date.strftime('%m')) + except: + return word("Bad date") + + +def day_of(the_date, language=None): + """Return the day-of-month component of a date. + + Args: + the_date (datetime.date, datetime.datetime, or str): The date to + extract the day from. + language (str or None): Unused; retained for API consistency. + + Returns: + int: Day of the month (1ā€“31). + """ + ensure_definition(the_date, language) + try: + if isinstance(the_date, (datetime.datetime, datetime.date)): + date = the_date + else: + date = dateutil.parser.parse(the_date) + return int(date.strftime('%d')) + except: + return word("Bad date") + + +def dow_of(the_date, as_word=False, language=None): + """Return the day of the week for a date. + + Args: + the_date (datetime.date, datetime.datetime, or str): The date to + inspect. + as_word (bool): If True, return the full weekday name (e.g. + ``'Monday'``); otherwise return an integer from 1 (Monday) to + 7 (Sunday) per ISO 8601. + language (str or None): Language code for localizing the weekday + name. Defaults to the current interview language. + + Returns: + int or str: Day-of-week number or localized weekday name. + """ + ensure_definition(the_date, as_word, language) + if language is None: + language = get_language() + try: + if isinstance(the_date, (datetime.datetime, datetime.date)): + date = the_date + else: + date = dateutil.parser.parse(the_date) + if as_word: + return babel.dates.format_date(date, format='EEEE', locale=babel_language(language)) + return int(date.strftime('%u')) + except: + return word("Bad date") + + +def year_of(the_date, language=None): + """Return the year component of a date. + + Args: + the_date (datetime.date, datetime.datetime, or str): The date to + extract the year from. + language (str or None): Unused; retained for API consistency. + + Returns: + int: Four-digit year. + """ + ensure_definition(the_date, language) + try: + if isinstance(the_date, (datetime.datetime, datetime.date)): + date = the_date + else: + date = dateutil.parser.parse(the_date) + return int(date.strftime('%Y')) + except: + return word("Bad date") + + +def format_date(the_date, format=None, language=None): # pylint: disable=redefined-builtin + """Return a date formatted as a localized string. + + Args: + the_date (datetime.date, datetime.datetime, or str): Date to format. + format (str or None): Babel date-format pattern (e.g. ``'long'``, + ``'short'``, ``'MM/dd/yyyy'``). Defaults to the interview's + configured date format or ``'long'``. + language (str or None): Language/locale code. Defaults to the current + interview language. + + Returns: + str: Formatted date string, or ``''`` for an empty date. + """ + ensure_definition(the_date, format, language) + if isinstance(the_date, DAEmpty): + return "" + if language is None: + language = get_language() + if format is None: + format = interview_default('date format', 'long', language) + try: + if isinstance(the_date, (datetime.datetime, datetime.date)): + date = the_date + else: + date = dateutil.parser.parse(the_date) + return babel.dates.format_date(date, format=format, locale=babel_language(language)) + except: + return word("Bad date") + + +def format_datetime(the_date, format=None, language=None): # pylint: disable=redefined-builtin + """Return a date and time formatted as a localized string. + + Args: + the_date (datetime.datetime or str): Date/time to format. + format (str or None): Babel datetime-format pattern. Defaults to the + interview's configured datetime format or ``'long'``. + language (str or None): Language/locale code. Defaults to the current + interview language. + + Returns: + str: Formatted datetime string, or ``''`` for an empty date. + """ + ensure_definition(the_date, format, language) + if isinstance(the_date, DAEmpty): + return "" + if language is None: + language = get_language() + if format is None: + format = interview_default('datetime format', 'long', language) + try: + if isinstance(the_date, (datetime.datetime, datetime.date)): + date = the_date + else: + date = dateutil.parser.parse(the_date) + return babel.dates.format_datetime(date, format=format, locale=babel_language(language)) + except: + return word("Bad date") + + +def format_time(the_time, format=None, language=None): # pylint: disable=redefined-builtin + """Return a time formatted as a localized string. + + Args: + the_time (datetime.time, datetime.datetime, or str): Time to format. + format (str or None): Babel time-format pattern. Defaults to the + interview's configured time format or ``'short'``. + language (str or None): Language/locale code. Defaults to the current + interview language. + + Returns: + str: Formatted time string, or ``''`` for an empty time. + """ + ensure_definition(the_time, format, language) + if isinstance(the_time, DAEmpty): + return "" + if language is None: + language = get_language() + if format is None: + format = interview_default('time format', 'short', language) + try: + if isinstance(the_time, (datetime.datetime, datetime.date, datetime.time)): + this_time = the_time + else: + this_time = dateutil.parser.parse(the_time) + return babel.dates.format_time(this_time, format=format, locale=babel_language(language)) + except BaseException as errmess: + return word("Bad date: " + str(errmess)) + + +class DateTimeDelta: + + def __str__(self): + return str(self.describe()) + + def describe(self, **kwargs): + specificity = kwargs.get('specificity', None) + output = [] + diff = dateutil.relativedelta.relativedelta(self.end, self.start) + if diff.years != 0: + output.append((abs(diff.years), noun_plural(word('year'), abs(diff.years), noun_is_singular=True))) + if diff.months != 0 and specificity != 'year': + output.append((abs(diff.months), noun_plural(word('month'), abs(diff.months), noun_is_singular=True))) + if diff.days != 0 and specificity not in ('year', 'month'): + output.append((abs(diff.days), noun_plural(word('day'), abs(diff.days), noun_is_singular=True))) + if len(output) == 0 or specificity in ('hour', 'minute', 'second'): + if diff.hours != 0 and specificity not in ('year', 'month', 'day'): + output.append((abs(diff.hours), noun_plural(word('hour'), abs(diff.hours), noun_is_singular=True))) + if (abs(diff.hours) < 2 or specificity in ('minute', 'second')) and diff.minutes != 0 and specificity not in ('year', 'month', 'day', 'hour'): + output.append((abs(diff.minutes), noun_plural(word('minute'), abs(diff.minutes), noun_is_singular=True))) + if len(output) == 0 or specificity == 'second': + if diff.seconds != 0 and specificity not in ('year', 'month', 'day', 'hour', 'minute'): + output.append((abs(diff.seconds), noun_plural(word('second'), abs(diff.seconds), noun_is_singular=True))) + if len(output) == 0: + if specificity is None: + output.append((0, noun_plural(word('second'), 0, noun_is_singular=True))) + else: + output.append((0, noun_plural(word(specificity), 0, noun_is_singular=True))) + if kwargs.get('nice', True): + return_value = comma_and_list(["%s %s" % (nice_number(y[0]), y[1]) for y in output]) + if kwargs.get('capitalize', False): + return capitalize(return_value) + return return_value + return comma_and_list(["%d %s" % y for y in output]) + + +class DADateTime(datetime.datetime): + """A timezone-aware datetime subclass with docassemble-specific formatting and arithmetic. + + Inherits all ``datetime.datetime`` behavior and adds convenience methods + for formatting, date arithmetic, and accessing ISO calendar properties. + + Attributes: + dow (int): Day of the week (1 = Monday ā€¦ 7 = Sunday, ISO 8601). + week (int): ISO week number of the year. + nanosecond (int): Always 0; provided for compatibility. + """ + + def format(self, format=None, language=None): # pylint: disable=redefined-builtin + return format_date(self, format=format, language=language) + + def format_date(self, format=None, language=None): # pylint: disable=redefined-builtin + return format_date(self, format=format, language=language) + + def format_datetime(self, format=None, language=None): # pylint: disable=redefined-builtin + return format_datetime(self, format=format, language=language) + + def format_time(self, format=None, language=None): # pylint: disable=redefined-builtin + return format_time(self, format=format, language=language) + + def replace_time(self, the_time): + return self.replace(hour=the_time.hour, minute=the_time.minute, second=the_time.second, microsecond=the_time.microsecond) + + @property + def nanosecond(self): + return 0 + + @property + def dow(self): + return self.isocalendar()[2] + + @property + def week(self): + return self.isocalendar()[1] + + def plus(self, **kwargs): + return dd(dt(self) + date_interval(**kwargs)) + + def minus(self, **kwargs): + return dd(dt(self) - date_interval(**kwargs)) + + def __str__(self): + return str(format_date(self)) + + def __add__(self, other): + if isinstance(other, str): + return str(self) + other + val = dt(self) + other + if isinstance(val, datetime.date): + return dd(val) + return val + + def __radd__(self, other): + if isinstance(other, str): + return other + str(self) + return dd(dt(self) + other) + + def __sub__(self, other): + val = dt(self) - other + if isinstance(val, datetime.date): + return dd(val) + return val + + def __rsub__(self, other): + val = other - dt(self) + if isinstance(val, datetime.date): + return dd(val) + return val + + +def current_datetime(timezone=None): + """Return the current date and time as a DADateTime object. + + Args: + timezone (str or None): IANA timezone name. If None, the interview's + default timezone is used. + + Returns: + DADateTime: Current date and time in the specified timezone. + """ + ensure_definition(timezone) + if timezone is None: + timezone = get_default_timezone() + return dd(datetime.datetime.now(datetime.timezone.utc).astimezone(zoneinfo.ZoneInfo(timezone))) + + +def as_datetime(the_date, timezone=None): + """Convert a date or date string to a timezone-aware DADateTime object. + + Args: + the_date (datetime.date, datetime.datetime, or str): Date or + date/time value to convert. String values are parsed with + ``dateutil``. + timezone (str or None): IANA timezone name to attach. If the value + already carries timezone information it is converted to this + zone; otherwise the timezone is applied as-is. Defaults to + the interview's default timezone. + + Returns: + DADateTime: Timezone-aware datetime. + """ + ensure_definition(the_date, timezone) + if timezone is None: + timezone = get_default_timezone() + if isinstance(the_date, datetime.date) and not isinstance(the_date, datetime.datetime): + the_date = datetime.datetime.combine(the_date, datetime.datetime.min.time()) + if isinstance(the_date, datetime.datetime): + new_datetime = the_date + else: + new_datetime = dateutil.parser.parse(the_date) + if new_datetime.tzinfo: + new_datetime = new_datetime.astimezone(zoneinfo.ZoneInfo(timezone)) + else: + new_datetime = new_datetime.replace(tzinfo=zoneinfo.ZoneInfo(timezone)) + return dd(new_datetime) + + +def dd(obj): + if isinstance(obj, DADateTime): + return obj + return DADateTime(obj.year, month=obj.month, day=obj.day, hour=obj.hour, minute=obj.minute, second=obj.second, microsecond=obj.microsecond, tzinfo=obj.tzinfo) + + +def dt(obj): + return datetime.datetime(obj.year, obj.month, obj.day, obj.hour, obj.minute, obj.second, obj.microsecond, obj.tzinfo) + + +def date_interval(**kwargs): + """Return a relative date/time interval. + + All keyword arguments are forwarded to + ``dateutil.relativedelta.relativedelta``. Common arguments include + ``years``, ``months``, ``weeks``, ``days``, ``hours``, ``minutes``, + and ``seconds``. + + Returns: + dateutil.relativedelta.relativedelta: Interval that can be added to + or subtracted from a ``DADateTime`` or ``datetime`` object. + """ + ensure_definition(**kwargs) + return dateutil.relativedelta.relativedelta(**kwargs) + + +def date_difference(starting=None, ending=None, timezone=None): + """Return the difference between two dates. + + Args: + starting (datetime.date, datetime.datetime, str, or None): Start of + the interval. Defaults to the current datetime. + ending (datetime.date, datetime.datetime, str, or None): End of the + interval. Defaults to the current datetime. + timezone (str or None): IANA timezone name used when localizing + naive datetimes. Defaults to the interview's default timezone. + + Returns: + DateTimeDelta: Object with ``weeks``, ``days``, ``hours``, + ``minutes``, ``seconds``, ``years``, and ``delta`` attributes + expressing the difference, and ``start``/``end`` attributes + holding the resolved datetime objects. + """ + ensure_definition(starting, ending, timezone) + if starting is None: + starting = current_datetime() + if ending is None: + ending = current_datetime() + if timezone is None: + timezone = get_default_timezone() + if isinstance(starting, datetime.date) and not isinstance(starting, datetime.datetime): + starting = datetime.datetime.combine(starting, datetime.datetime.min.time()) + if isinstance(ending, datetime.date) and not isinstance(ending, datetime.datetime): + ending = datetime.datetime.combine(ending, datetime.datetime.min.time()) + if not isinstance(starting, datetime.datetime): + starting = dateutil.parser.parse(starting) + if not isinstance(ending, datetime.datetime): + ending = dateutil.parser.parse(ending) + if starting.tzinfo: + starting = starting.astimezone(zoneinfo.ZoneInfo(timezone)) + else: + starting = starting.replace(tzinfo=zoneinfo.ZoneInfo(timezone)) + if ending.tzinfo: + ending = ending.astimezone(zoneinfo.ZoneInfo(timezone)) + else: + ending = ending.replace(tzinfo=zoneinfo.ZoneInfo(timezone)) + delta = ending - starting + output = DateTimeDelta() + output.start = starting + output.end = ending + output.weeks = (delta.days / 7.0) + (delta.seconds / 604800.0) + output.days = delta.days + (delta.seconds / 86400.0) + output.hours = (delta.days * 24.0) + (delta.seconds / 3600.0) + output.minutes = (delta.days * 1440.0) + (delta.seconds / 60.0) + output.seconds = (delta.days * 86400) + delta.seconds + output.years = (delta.days + delta.seconds / 86400.0) / 365.2425 + output.delta = delta + return output diff --git a/docassemble_base/docassemble/base/empty.py b/docassemble_base/docassemble/base/empty.py new file mode 100644 index 000000000..2fb2f1f68 --- /dev/null +++ b/docassemble_base/docassemble/base/empty.py @@ -0,0 +1,199 @@ +class DAEmpty: + """An object that silently absorbs any attribute access or operation. + + DAEmpty avoids triggering errors about missing information by returning + another DAEmpty for any attribute access, returning empty values for + string conversion and length, and absorbing arithmetic operations. + + Attributes: + str (str): The string value returned when the object is converted to + text. Defaults to the empty string. + """ + + def __init__(self, *pargs, **kwargs): # pylint: disable=unused-argument + self.str = str(kwargs.get('str', '')) + + def __getattr__(self, thename): + if thename.startswith('__') or thename == 'str': + return object.__getattribute__(self, thename) + return DAEmpty() + + def __str__(self): + try: + return object.__getattribute__(self, 'str') + except: + return '' + + def __dir__(self): + return [] + + def __contains__(self, item): + return False + + def __iter__(self): + the_list = [] + return the_list.__iter__() + + def __len__(self): + return 0 + + def __reversed__(self): + return [] + + def __getitem__(self, index): + return DAEmpty() + + def __setitem__(self, index, val): + pass + + def __delitem__(self, index): + pass + + def __call__(self, *pargs, **kwargs): + return DAEmpty() + + def __repr__(self): + return repr('') + + def __add__(self, other): + return other + + def __sub__(self, other): + return other + + def __mul__(self, other): + return other + + def __floordiv__(self, other): + return other + + def __mod__(self, other): + return other + + def __divmod__(self, other): + return other + + def __pow__(self, other): + return other + + def __lshift__(self, other): + return other + + def __rshift__(self, other): + return other + + def __and__(self, other): + return other + + def __xor__(self, other): + return other + + def __or__(self, other): + return other + + def __div__(self, other): + return other + + def __truediv__(self, other): + return other + + def __radd__(self, other): + return other + + def __rsub__(self, other): + return other + + def __rmul__(self, other): + return other + + def __rdiv__(self, other): + return other + + def __rtruediv__(self, other): + return other + + def __rfloordiv__(self, other): + return other + + def __rmod__(self, other): + return other + + def __rdivmod__(self, other): + return other + + def __rpow__(self, other): + return other + + def __rlshift__(self, other): + return other + + def __rrshift__(self, other): + return other + + def __rand__(self, other): + return other + + def __ror__(self, other): + return other + + def __neg__(self): + return 0 + + def __pos__(self): + return 0 + + def __abs__(self): + return 0 + + def __invert__(self): + return 0 + + def __complex__(self): + return 0 + + def __int__(self): + return int(0) + + def __float__(self): + return float(0) + + def __oct__(self): + return oct(0) + + def __hex__(self): + return hex(0) + + def __index__(self): + return int(0) + + def __le__(self, other): + return True + + def __ge__(self, other): + return self is other or False + + def __gt__(self, other): + return False + + def __lt__(self, other): + return True + + def __eq__(self, other): + return self is other + + def __ne__(self, other): + return self is not other + + def __hash__(self): + return hash(('',)) + + def as_dict(self): + return self.to_json() + + def to_json(self): + output = {'_class': 'docassemble.base.util.DAEmpty'} + try: + output.update({'str': object.__getattribute__(self, 'str')}) + except Exception: + pass + return output diff --git a/docassemble_base/docassemble/base/error.py b/docassemble_base/docassemble/base/error.py index 487086130..97a7b2d4e 100644 --- a/docassemble_base/docassemble/base/error.py +++ b/docassemble_base/docassemble/base/error.py @@ -106,7 +106,7 @@ def __init__(self, *pargs, **kwargs): if len(the_args) == 0: raise DAError("ForcedNameError must have at least one argument") the_context = {} - the_user_dict = kwargs.get('user_dict', {}) + the_user_dict = kwargs.get('user_dict') or {} for var_name in ('x', 'i', 'j', 'k', 'l', 'm', 'n'): if var_name in the_user_dict: the_context[var_name] = the_user_dict[var_name] diff --git a/docassemble_base/docassemble/base/file_docx.py b/docassemble_base/docassemble/base/file_docx.py index bc19316a5..2b0ff0e22 100644 --- a/docassemble_base/docassemble/base/file_docx.py +++ b/docassemble_base/docassemble/base/file_docx.py @@ -1,92 +1,30 @@ -import re import os -import codecs import time import stat import mimetypes import tempfile -import string import shutil import zipfile from collections import deque -from copy import deepcopy from xml.sax.saxutils import escape as html_escape -from docxtpl import InlineImage, RichText -from docx.shared import Mm, Inches, Pt, Cm, Twips import docx.opc.constants -from docx.oxml.section import CT_SectPr -from docx.oxml.table import CT_Tbl import docx from docxcompose.composer import Composer # For fixing up images, etc when including docx files within templates -import docassemble.base.functions -from docassemble.base.functions import server, package_template_filename, get_config, roman -from docassemble.base.error import DAError -import docassemble.base.filter -import docassemble.base.pandoc -from docassemble.base.logger import logmessage -from bs4 import BeautifulSoup, NavigableString, Tag +from bs4 import NavigableString from pikepdf import Pdf - -zerowidth = '\u200B' +from .error import DAError +from .filter.image_docx import image_for_docx +from .filter.utils import sanitize_xml +from .functions import DALocalFile +from .hooks import fg_make_pdf_for_word_path, fg_make_png_for_pdf_path +from .pandoc import convert_file +from .thread_context import this_thread QPDF_PATH = 'qpdf' NoneType = type(None) DEFAULT_PAGE_WIDTH = '6.5in' -list_types = ['1', 'A', 'a', 'I', 'i'] - - -def fix_double_quote(the_string): - return '"' + re.sub('"', '"', the_string) + '"' - - -class CustomInlineImage(InlineImage): - alt_text = None - - def __init__(self, tpl, image_descriptor, width=None, height=None, anchor=None, alt_text=None): - super().__init__(tpl, image_descriptor, width=width, height=height, anchor=anchor) - self.alt_text = alt_text - - def _insert_image(self): - output = super()._insert_image() - if self.alt_text: - return re.sub(']*>\s*(.*)\s*', r'\1', sanitize_xml(str(first_paragraph._p.xml)), flags=re.DOTALL) - return sanitize_xml(str(sd)) - - for key, val in kwargs.items(): - if hasattr(val, 'instanceName'): - the_repr = val.instanceName - elif isinstance(val, (int, float, bool, NoneType)): - the_repr = val - else: - the_repr = '_codecs.decode(_array.array("b", "' + re.sub(r'\n', '', codecs.encode(bytearray(val, encoding='utf-8'), 'base64').decode()) + '".encode()), "base64").decode()' - first_paragraph.insert_paragraph_before(str("{%%p set %s = %s %%}" % (key, the_repr))) - if 'docx_include_count' not in docassemble.base.functions.this_thread.misc: - docassemble.base.functions.this_thread.misc['docx_include_count'] = 0 - docassemble.base.functions.this_thread.misc['docx_include_count'] += 1 - if single_paragraph: - return re.sub(r']*>\s*(.*)\s*', r'\1', str(first_paragraph._p.xml), flags=re.DOTALL) - return sd - - def get_children(descendants, parsed): subelement = False descendants_buff = deque() @@ -238,489 +88,13 @@ def html_linear_parse(soup): return parsed -def Alpha(number): - multiplier = int((number - 1) / 26) - indexno = (number - 1) % 26 - return string.ascii_uppercase[indexno] * (multiplier + 1) - - -def alpha(number): - multiplier = int((number - 1) / 26) - indexno = (number - 1) % 26 - return string.ascii_lowercase[indexno] * (multiplier + 1) - - -def Roman_Numeral(number): - return roman((number - 1) % 4000, case='upper') - - -def roman_numeral(number): - return roman((number - 1) % 4000, case='lower') - - -class SoupParser: - - def __init__(self, tpl): - self.paragraphs = [{'params': {'style': 'p', 'indentation': 0, 'list_number': 1}, 'runs': [RichText('')]}] - self.current_paragraph = self.paragraphs[-1] - self.run = self.current_paragraph['runs'][-1] - self.bold = False - self.center = False - self.list_number = 1 - self.list_type = list_types[-1] - self.italic = False - self.underline = False - self.strike = False - self.indentation = 0 - self.style = 'p' - self.still_new = True - self.size = None - self.charstyle = None - self.color = None - self.tpl = tpl - - def new_paragraph(self, classes, styles): - if self.still_new: - # logmessage("new_paragraph is still new and style is " + self.style + " and indentation is " + str(self.indentation)) - self.current_paragraph['params']['style'] = self.style - self.current_paragraph['params']['indentation'] = self.indentation - self.set_attribs(classes, styles) - self.list_number += 1 - return - # logmessage("new_paragraph where style is " + self.style + " and indentation is " + str(self.indentation)) - self.current_paragraph = {'params': {'style': self.style, 'indentation': self.indentation, 'list_number': self.list_number}, 'runs': [RichText('')]} - self.set_attribs(classes, styles) - self.list_number += 1 - self.paragraphs.append(self.current_paragraph) - self.run = self.current_paragraph['runs'][-1] - self.still_new = True - - def set_attribs(self, classes, styles): - if 'dacenter' in classes: - self.current_paragraph['params']['align'] = 'center' - elif 'daflushright' in classes: - self.current_paragraph['params']['align'] = 'end' - else: - self.current_paragraph['params']['align'] = 'start' - if len(classes): - if 'daspacingtight' in classes: - self.current_paragraph['params']['spacing'] = 240 - self.current_paragraph['params']['after'] = 0 - elif 'daspacingsingle' in classes: - self.current_paragraph['params']['spacing'] = 240 - self.current_paragraph['params']['after'] = 240 - elif 'daspacingdouble' in classes: - self.current_paragraph['params']['spacing'] = 480 - self.current_paragraph['params']['after'] = 0 - elif 'daspacingoneandahalf' in classes: - self.current_paragraph['params']['spacing'] = 260 - self.current_paragraph['params']['after'] = 0 - elif 'daspacingtriple' in classes: - self.current_paragraph['params']['spacing'] = 700 - self.current_paragraph['params']['after'] = 0 - if styles: - m = re.search(r'margin-left:([0-9\.]+)px', styles) - if m: - self.current_paragraph['params']['leftindent'] = 20 * int(m.group(1)) - m = re.search(r'margin-right:([0-9\.]+)px', styles) - if m: - self.current_paragraph['params']['rightindent'] = 20 * int(m.group(1)) - m = re.search(r'text-indent:([0-9\.]+)px', styles) - if m: - self.current_paragraph['params']['firstline'] = 20 * int(m.group(1)) - - def __str__(self): - output = '' - for para in self.paragraphs: - # logmessage("Got a paragraph where style is " + para['params'].get('style', 'undefined') + " and indentation is " + str(para['params'].get('indentation', 'undefined'))) - output += '' - if 'align' not in para['params']: - para['params']['align'] = 'start' - if para['params']['align'] == 'center': - output += '' - elif para['params']['align'] == 'end': - output += '' - if 'spacing' in para['params']: - output += '' - if para['params']['style'] == 'ul' or para['params']['style'].startswith('ol'): - if 'leftindent' in para['params']: - left_indent = para['params']['leftindent'] - else: - left_indent = 36*para['params']['indentation'] - if 'rightindent' in para['params']: - right_indent = para['params']['rightindent'] - else: - right_indent = 0 - output += '' - elif para['params']['style'] == 'blockquote': - if 'spacing' not in para['params']: - output += '' - output += '' - elif 'leftindent' in para['params'] or 'rightindent' in para['params'] or 'firstline' in para['params']: - if 'leftindent' in para['params']: - left_indent = para['params']['leftindent'] - else: - left_indent = 0 - if 'rightindent' in para['params']: - right_indent = para['params']['rightindent'] - else: - right_indent = 0 - if 'firstline' in para['params']: - first_line = para['params']['firstline'] - else: - first_line = 0 - output += '' - output += '' - if para['params']['style'] == 'ul': - output += str(RichText("ā€¢\t")) - if para['params']['style'] == 'ol1': - output += str(RichText(str(para['params']['list_number']) + ".\t")) - elif para['params']['style'] == 'olA': - output += str(RichText(Alpha(para['params']['list_number']) + ".\t")) - elif para['params']['style'] == 'ola': - output += str(RichText(alpha(para['params']['list_number']) + ".\t")) - elif para['params']['style'] == 'olI': - output += str(RichText(Roman_Numeral(para['params']['list_number']) + ".\t")) - elif para['params']['style'] == 'oli': - output += str(RichText(roman_numeral(para['params']['list_number']) + ".\t")) - for run in para['runs']: - output += str(run) - output += '' - return output - - def start_link(self, url): - ref = self.tpl.docx._part.relate_to(url, docx.opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external=True) - self.current_paragraph['runs'].append('' % (ref, )) - self.new_run() - self.still_new = False - - def end_link(self): - self.current_paragraph['runs'].append('') - self.new_run() - self.still_new = False - - def new_run(self): - self.current_paragraph['runs'].append(RichText('')) - self.run = self.current_paragraph['runs'][-1] - - def traverse(self, elem): - for part in elem.contents: - if isinstance(part, NavigableString): - self.run.add(str(part), italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) - self.still_new = False - elif isinstance(part, Tag): - # logmessage("Part name is " + str(part.name)) - if part.name == 'p': - if 'class' in part.attrs: - classes = part.attrs['class'] - else: - classes = [] - if 'style' in part.attrs: - styles = part.attrs['style'] - else: - styles = "" - self.new_paragraph(classes, styles) - if 'dabold' in classes: - self.bold = True - self.traverse(part) - if 'dabold' in classes: - self.bold = False - elif part.name == 'li': - if 'class' in part.attrs: - classes = part.attrs['class'] - else: - classes = [] - if 'style' in part.attrs: - styles = part.attrs['style'] - else: - styles = "" - self.new_paragraph(classes, styles) - self.traverse(part) - elif part.name == 'ul': - # logmessage("Entering a UL") - oldstyle = self.style - self.style = 'ul' - self.indentation += 10 - self.traverse(part) - self.indentation -= 10 - self.style = oldstyle - # logmessage("Leaving a UL") - elif part.name == 'ol': - # logmessage("Entering a OL") - oldstyle = self.style - oldlistnumber = self.list_number - oldlisttype = self.list_type - if part.get('type', None) in list_types: - self.list_type = part['type'] - else: - self.list_type = list_types[(list_types.index(self.list_type) + 1) % 5] - try: - self.list_number = int(part.get('start', 1)) - except: - self.list_number = 1 - self.style = 'ol' + self.list_type - self.indentation += 10 - self.traverse(part) - self.indentation -= 10 - self.list_type = oldlisttype - self.list_number = oldlistnumber - self.style = oldstyle - # logmessage("Leaving a OL") - elif part.name == 'strong': - self.bold = True - self.traverse(part) - self.bold = False - elif part.name == 'em': - self.italic = True - self.traverse(part) - self.italic = False - elif part.name == 'strike': - self.strike = True - self.traverse(part) - self.strike = False - elif part.name == 'u': - self.underline = True - self.traverse(part) - self.underline = False - elif part.name == 'blockquote': - oldstyle = self.style - self.style = 'blockquote' - self.indentation += 20 - self.traverse(part) - self.indentation -= 20 - self.style = oldstyle - elif re.match(r'h[1-6]', part.name): - oldsize = self.size - self.size = 60 - ((int(part.name[1]) - 1) * 10) - if 'class' in part.attrs: - classes = part.attrs['class'] - else: - classes = [] - if 'style' in part.attrs: - styles = part.attrs['style'] - else: - styles = "" - self.new_paragraph(classes, styles) - self.bold = True - self.traverse(part) - self.bold = False - self.size = oldsize - elif part.name == 'a': - self.start_link(part['href']) - if self.tpl.da_hyperlink_style: - self.charstyle = self.tpl.da_hyperlink_style - else: - self.underline = True - self.color = '#0000ff' - self.traverse(part) - if self.tpl.da_hyperlink_style: - self.charstyle = None - else: - self.underline = False - self.color = None - self.end_link() - elif part.name == 'br': - self.run.add("\n", italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) - self.still_new = False - else: - logmessage("Encountered a " + part.__class__.__name__) - - -class InlineSoupParser: - - def __init__(self, tpl): - self.runs = [RichText('')] - self.run = self.runs[-1] - self.bold = False - self.italic = False - self.underline = False - self.indentation = 0 - self.style = 'p' - self.strike = False - self.size = None - self.charstyle = None - self.color = None - self.tpl = tpl - self.at_start = True - self.list_number = 1 - self.list_type = list_types[-1] - - def new_paragraph(self): - if self.at_start: - self.at_start = False - else: - self.run.add("\n", italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) - if self.indentation: - self.run.add("\t" * self.indentation) - if self.style == 'ul': - self.run.add("ā€¢\t") - if self.style == 'ol1': - self.run.add(str(self.list_number) + ".\t") - self.list_number += 1 - elif self.style == 'olA': - self.run.add(Alpha(self.list_number) + ".\t") - self.list_number += 1 - elif self.style == 'ola': - self.run.add(alpha(self.list_number) + ".\t") - self.list_number += 1 - elif self.style == 'olI': - self.run.add(Roman_Numeral(self.list_number) + ".\t") - self.list_number += 1 - elif self.style == 'oli': - self.run.add(roman_numeral(self.list_number) + ".\t") - self.list_number += 1 - # else: - # self.list_number = 1 - - def __str__(self): - output = '' - for run in self.runs: - output += str(run) - return output - - def start_link(self, url): - ref = self.tpl.docx._part.relate_to(url, docx.opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external=True) - self.runs.append('' % (ref, )) - self.new_run() - - def end_link(self): - self.runs.append('') - self.new_run() - - def new_run(self): - self.runs.append(RichText('')) - self.run = self.runs[-1] - - def traverse(self, elem): - for part in elem.contents: - if isinstance(part, NavigableString): - self.run.add(str(part), italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) - elif isinstance(part, Tag): - if part.name in ('p', 'blockquote'): - self.new_paragraph() - self.traverse(part) - elif part.name == 'li': - self.new_paragraph() - self.traverse(part) - elif part.name == 'ul': - oldstyle = self.style - self.style = 'ul' - self.indentation += 1 - self.traverse(part) - self.indentation -= 1 - self.style = oldstyle - elif part.name == 'ol': - oldstyle = self.style - oldlistnumber = self.list_number - oldlisttype = self.list_type - if part.get('type', None) in list_types: - self.list_type = part['type'] - else: - self.list_type = list_types[(list_types.index(self.list_type) + 1) % 5] - try: - self.list_number = int(part.get('start', 1)) - except: - self.list_number = 1 - self.style = 'ol' + self.list_type - self.indentation += 1 - self.traverse(part) - self.indentation -= 1 - self.list_type = oldlisttype - self.list_number = oldlistnumber - self.style = oldstyle - elif part.name == 'strong': - self.bold = True - self.traverse(part) - self.bold = False - elif part.name == 'em': - self.italic = True - self.traverse(part) - self.italic = False - elif part.name == 'strike': - self.strike = True - self.traverse(part) - self.strike = False - elif part.name == 'u': - self.underline = True - self.traverse(part) - self.underline = False - elif re.match(r'h[1-6]', part.name): - oldsize = self.size - self.size = 60 - ((int(part.name[1]) - 1) * 10) - self.bold = True - self.traverse(part) - self.bold = False - self.size = oldsize - elif part.name == 'a': - self.start_link(part['href']) - if self.tpl.da_hyperlink_style: - self.charstyle = self.tpl.da_hyperlink_style - else: - self.underline = True - self.color = '#0000ff' - self.traverse(part) - if self.tpl.da_hyperlink_style: - self.charstyle = None - else: - self.underline = False - self.color = None - self.end_link() - elif part.name == 'br': - self.run.add("\n", italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) - else: - logmessage("Encountered a " + part.__class__.__name__) - - -def inline_markdown_to_docx(text, question, tpl): - old_context = docassemble.base.functions.this_thread.evaluation_context - docassemble.base.functions.this_thread.evaluation_context = None - try: - text = str(text) - except: - docassemble.base.functions.this_thread.evaluation_context = old_context - raise - docassemble.base.functions.this_thread.evaluation_context = old_context - source_code = docassemble.base.filter.markdown_to_html(text, do_terms=False) - source_code = re.sub(r"\n", ' ', source_code) - source_code = re.sub(r">\s+<", '><', source_code) - soup = BeautifulSoup('' + source_code + '', 'html.parser') - parser = InlineSoupParser(tpl) - for elem in soup.find_all(recursive=False): - parser.traverse(elem) - output = str(parser) - return docassemble.base.filter.docx_template_filter(output, question=question, replace_newlines=False) - - -def markdown_to_docx(text, question, tpl): - old_context = docassemble.base.functions.this_thread.evaluation_context - docassemble.base.functions.this_thread.evaluation_context = None - try: - text = str(text) - except: - docassemble.base.functions.this_thread.evaluation_context = old_context - raise - docassemble.base.functions.this_thread.evaluation_context = old_context - if get_config('new markdown to docx', False): - source_code = docassemble.base.filter.markdown_to_html(text, do_terms=False) - source_code = re.sub(r"\n", ' ', source_code) - source_code = re.sub(r">\s+<", '><', source_code) - soup = BeautifulSoup('' + source_code + '', 'html.parser') - parser = SoupParser(tpl) - for elem in soup.find_all(recursive=False): - parser.traverse(elem) - output = str(parser) - # logmessage(output) - return docassemble.base.filter.docx_template_filter(output, question=question) - return inline_markdown_to_docx(text, question, tpl) - - def pdf_pages(file_info, width): output = '' if width is None: width = DEFAULT_PAGE_WIDTH if not os.path.isfile(file_info['path'] + '.pdf'): if file_info['extension'] in ('rtf', 'doc', 'odt') and not os.path.isfile(file_info['path'] + '.pdf'): - server.fg_make_pdf_for_word_path(file_info['path'], file_info['extension']) + fg_make_pdf_for_word_path(file_info['path'], file_info['extension']) if 'pages' not in file_info: try: with Pdf.open(file_info['path'] + '.pdf') as reader: @@ -741,9 +115,9 @@ def pdf_pages(file_info, width): page_file['path'] = file_info['path'] + 'page-' + formatter % page page_file['fullpath'] = page_file['path'] + '.png' if not os.path.isfile(page_file['fullpath']): - server.fg_make_png_for_pdf_path(file_info['path'] + '.pdf', 'page') + fg_make_png_for_pdf_path(file_info['path'] + '.pdf', 'page') if os.path.isfile(page_file['fullpath']): - output += str(image_for_docx(docassemble.base.functions.DALocalFile(page_file['fullpath']), docassemble.base.functions.this_thread.current_question, docassemble.base.functions.this_thread.misc.get('docx_template', None), width=width)) + output += str(image_for_docx(DALocalFile(page_file['fullpath']), this_thread.current_question, this_thread.misc.get('docx_template', None), width=width)) else: output += "[Error including page image]" output += ' ' @@ -762,7 +136,7 @@ def concatenate_files(path_list): ext = 'doc' else: ext = 'odt' - docassemble.base.pandoc.convert_file(path, new_docx_file.name, ext, 'docx') + convert_file(path, new_docx_file.name, ext, 'docx') new_path_list.append(new_docx_file.name) elif mimetype == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': new_path_list.append(path) @@ -778,10 +152,6 @@ def concatenate_files(path_list): return docx_file.name -def sanitize_xml(text): - return re.sub(r'{([{%#])', '{' + zerowidth + r'\1', re.sub(r'([}%#])}', r'\1' + zerowidth + '}', text)) - - def fix_docx(path): seen = set() problem_present = False diff --git a/docassemble_base/docassemble/base/filter.py b/docassemble_base/docassemble/base/filter.py deleted file mode 100644 index b6b4ae893..000000000 --- a/docassemble_base/docassemble/base/filter.py +++ /dev/null @@ -1,2006 +0,0 @@ -import re -import os -import mimetypes -import codecs -import json -from io import BytesIO -import tempfile -import time -import stat -import xml.etree.ElementTree as ET -import qrcode -import qrcode.image.svg -from pikepdf import Pdf -import PIL -from docassemble.base.logger import logmessage -from docassemble.base.rtfng.object.picture import Image -from docassemble.base.functions import server, word -import docassemble.base.functions -from docassemble.base import pandoc -from bs4 import BeautifulSoup -from pylatex.utils import escape_latex -from cairosvg import svg2png, svg2eps - -QPDF_PATH = 'qpdf' -NoneType = type(None) - -zerowidth = '\u200B' - -DEFAULT_PAGE_WIDTH = '6.5in' - -term_start = re.compile(r'\[\[') -term_match = re.compile(r'\[\[([^\[\]\|]*)(\|[^\[\]]*)?\]\]', re.DOTALL) -noquote_match = re.compile(r'"') -lt_match = re.compile(r'<') -gt_match = re.compile(r'>') -amp_match = re.compile(r'&') -# amp_match = re.compile(r'&(?!#?[0-9A-Za-z]+;)') -emoji_match = re.compile(r':([A-Za-z][A-Za-z0-9\_\-]+):') -extension_match = re.compile(r'\.[a-z]+$') -map_match = re.compile(r'\[MAP ([^\]]+)\]', flags=re.DOTALL) -code_match = re.compile(r'') - - -def set_default_page_width(width): - global DEFAULT_PAGE_WIDTH - DEFAULT_PAGE_WIDTH = str(width) - - -def get_default_page_width(): - return DEFAULT_PAGE_WIDTH - -DEFAULT_IMAGE_WIDTH = '4in' - - -def set_default_image_width(width): - global DEFAULT_IMAGE_WIDTH - DEFAULT_IMAGE_WIDTH = str(width) - - -def get_default_image_width(): - return DEFAULT_IMAGE_WIDTH - -MAX_HEIGHT_POINTS = 10 * 72 - - -def set_max_height_points(points): - global MAX_HEIGHT_POINTS - MAX_HEIGHT_POINTS = points - - -def get_max_height_points(): - return MAX_HEIGHT_POINTS - -MAX_WIDTH_POINTS = 6.5 * 72.0 - - -def set_max_width_points(points): - global MAX_WIDTH_POINTS - MAX_WIDTH_POINTS = points - - -def get_max_width_points(): - return MAX_WIDTH_POINTS - -# def blank_da_send_mail(*args, **kwargs): -# logmessage("da_send_mail: no mail agent configured!") -# return(None) - -# da_send_mail = blank_da_send_mail - -# def set_da_send_mail(func): -# global da_send_mail -# da_send_mail = func -# return - -# def blank_file_finder(*args, **kwargs): -# return({'filename': "invalid"}) - -# file_finder = blank_file_finder - -# def set_file_finder(func): -# global file_finder -# #logmessage("set the file finder to " + str(func)) -# file_finder = func -# return - -# def blank_url_finder(*args, **kwargs): -# return('about:blank') - -# url_finder = blank_url_finder - -# def set_url_finder(func): -# global url_finder -# url_finder = func -# return - -# def blank_url_for(*args, **kwargs): -# return('about:blank') - -# url_for = blank_url_for - -# def set_url_for(func): -# global url_for -# url_for = func -# return - -rtf_spacing = {'tight': r'\\sl0 ', 'single': r'\\sl0 ', 'oneandahalf': r'\\sl360\\slmult1 ', 'double': r'\\sl480\\slmult1 ', 'triple': r'\\sl720\\slmult1 '} - -rtf_after_space = {'tight': 0, 'single': 1, 'oneandahalf': 0, 'double': 0, 'triplespacing': 0, 'triple': 0} - - -def rtf_prefilter(text): - text = re.sub(r'^# ', '[HEADING1] ', text, flags=re.MULTILINE) - text = re.sub(r'^## ', '[HEADING2] ', text, flags=re.MULTILINE) - text = re.sub(r'^### ', '[HEADING3] ', text, flags=re.MULTILINE) - text = re.sub(r'^#### ', '[HEADING4] ', text, flags=re.MULTILINE) - text = re.sub(r'^##### ', '[HEADING5] ', text, flags=re.MULTILINE) - text = re.sub(r'^###### ', '[HEADING6] ', text, flags=re.MULTILINE) - text = re.sub(r'^####### ', '[HEADING7] ', text, flags=re.MULTILINE) - text = re.sub(r'^######## ', '[HEADING8] ', text, flags=re.MULTILINE) - text = re.sub(r'^######### ', '[HEADING9] ', text, flags=re.MULTILINE) - text = re.sub(r'\s*\[VERTICAL_LINE\]\s*', '\n\n[VERTICAL_LINE]\n\n', text) - text = re.sub(r'\s*\[BREAK\]\s*', '\n\n[BREAK]\n\n', text) - text = re.sub(r'\s+\[END_TWOCOL\]', '\n\n[END_TWOCOL]', text) - text = re.sub(r'\s+\[END_CAPTION\]', '\n\n[END_CAPTION]', text) - text = re.sub(r'\[BEGIN_TWOCOL\]\s+', '[BEGIN_TWOCOL]\n\n', text) - text = re.sub(r'\[BEGIN_CAPTION\]\s+', '[BEGIN_CAPTION]\n\n', text) - return text - - -def repeat_along(chars, match): - output = chars * len(match.group(1)) - # logmessage("Output is " + repr(output)) - return output - - -def rtf_filter(text, metadata=None, styles=None, question=None): - if metadata is None: - metadata = {} - if styles is None: - styles = {} - # logmessage(text) - if 'fontsize' in metadata: - text = re.sub(r'{\\pard', r'\\fs' + str(convert_length(metadata['fontsize'], 'hp')) + r' {\\pard', text, count=1) - after_space_multiplier = convert_length(metadata['fontsize'], 'twips') - else: - after_space_multiplier = 240 - if 'IndentationAmount' in metadata: - indentation_amount = str(convert_length(metadata['IndentationAmount'], 'twips')) - else: - indentation_amount = '720' - if 'Indentation' in metadata: - default_indentation = bool(metadata['Indentation']) - else: - default_indentation = True - if 'SingleSpacing' in metadata and metadata['SingleSpacing']: - # logmessage("Gi there!") - default_spacing = 'single' - if 'Indentation' not in metadata: - default_indentation = False - elif 'OneAndAHalfSpacing' in metadata and metadata['OneAndAHalfSpacing']: - default_spacing = 'oneandahalf' - elif 'DoubleSpacing' in metadata and metadata['DoubleSpacing']: - default_spacing = 'double' - elif 'TripleSpacing' in metadata and metadata['TripleSpacing']: - default_spacing = 'triple' - else: - default_spacing = 'double' - after_space = after_space_multiplier * rtf_after_space[default_spacing] - text = re.sub(r'{\\pard \\ql \\f0 \\sa180 \\li0 \\fi0 \[HEADING([0-9]+)\] *', (lambda x: '{\\pard ' + styles.get(x.group(1), '\\ql \\f0 \\sa180 \\li0 \\fi0 ')), text) - text = re.sub(r'{\\pard \\ql \\f0 \\sa180 \\li0 \\fi0 \[(BEGIN_TWOCOL|BREAK|END_TWOCOL|BEGIN_CAPTION|VERTICAL_LINE|END_CAPTION|TIGHTSPACING|SINGLESPACING|DOUBLESPACING|START_INDENTATION|STOP_INDENTATION|PAGEBREAK|SKIPLINE|NOINDENT|FLUSHLEFT|FLUSHRIGHT|CENTER|BOLDCENTER|INDENTBY[^\]]*)\] *', r'[\1]{\\pard \\ql \\f0 \\sa180 \\li0 \\fi0 ', text) - text = re.sub(r'{\\pard \\ql \\f0 \\sa180 \\li0 \\fi0 *\\par}', r'', text) - text = re.sub(r'\[\[([^\]]*)\]\]', r'\1', text) - # with open('/tmp/asdf.rtf', 'w') as deb_file: - # deb_file.write(text) - text = re.sub(r'\\par}\s*\[(END_TWOCOL|END_CAPTION|BREAK|VERTICAL_LINE)\]', r'}[\1]', text, flags=re.DOTALL) - text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\s*\[BREAK\]\s*(.+?)\[END_TWOCOL\]', rtf_two_col, text, flags=re.DOTALL) - text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_as_rtf(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_as_rtf(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_as_rtf(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_as_rtf(x, question=question), text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_as_rtf, text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_as_rtf, text) - text = re.sub(r'\[QR ([^\]]+)\]', qr_as_rtf, text) - text = re.sub(r'\[MAP ([^\]]+)\]', '', text) - text = replace_fields(text) - # text = re.sub(r'\[FIELD ([^\]]+)\]', '', text) - text = re.sub(r'\[TARGET ([^\]]+)\]', '', text) - text = re.sub(r'\[YOUTUBE[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\[VIMEO[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\[BEGIN_CAPTION\](.+?)\s*\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', rtf_caption_table, text, flags=re.DOTALL) - text = re.sub(r'\[NBSP\]', r'\\~ ', text) - text = re.sub(r'\[REDACTION_SPACE\]', r'\\u9608\\zwbo', text) - text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('\\u9608', x), text) - text = re.sub(r'\[ENDASH\]', r'{\\endash}', text) - text = re.sub(r'\[EMDASH\]', r'{\\emdash}', text) - text = re.sub(r'\[HYPHEN\]', r'-', text) - text = re.sub(r'\[CHECKBOX\]', r'____', text) - text = re.sub(r'\[BLANK\]', r'________________', text) - text = re.sub(r'\[BLANKFILL\]', r'________________', text) - text = re.sub(r'\[PAGEBREAK\] *', r'\\page ', text) - text = re.sub(r'\[PAGENUM\]', r'{\\chpgn}', text) - text = re.sub(r'\[TOTALPAGES\]', r'{\\field{\\*\\fldinst NUMPAGES } {\\fldrslt 1}}', text) - text = re.sub(r'\[SECTIONNUM\]', r'{\\sectnum}', text) - text = re.sub(r' *\[SKIPLINE\] *', r'\\line ', text) - text = re.sub(r' *\[NEWLINE\] *', r'\\line ', text) - text = re.sub(r' *\[NEWPAR\] *', r'\\par ', text) - text = re.sub(r' *\[BR\] *', r'\\line ', text) - text = re.sub(r' *\[TAB\] *', r'\\tab ', text) - text = re.sub(r' *\[END\] *', r'\n', text) - text = re.sub(r'\\sa180\\sa180\\par', r'\\par', text) - text = re.sub(r'\\sa180', r'\\sa0', text) - text = re.sub(r'(\\trowd \\trgaph[0-9]+)', r'\1\\trqc', text) - text = re.sub(r'\\intbl\\row}\s*{\\pard', r'\\intbl\\row}\n\\line\n{\\pard', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\s*\[(SAVE|RESTORE|TIGHTSPACING|SINGLESPACING|DOUBLESPACING|TRIPLESPACING|ONEANDAHALFSPACING|START_INDENTATION|STOP_INDENTATION)\]\s*', r'\n[\1]\n', text) - lines = text.split('\n') - spacing_command = rtf_spacing[default_spacing] - if default_indentation: - indentation_command = r'\\fi' + str(indentation_amount) + " " - else: - indentation_command = r'\\fi0 ' - text = '' - formatting_stack = [] - for line in lines: - if re.search(r'\[SAVE\]', line): - formatting_stack.append({'spacing_command': spacing_command, 'after_space': after_space, 'default_indentation': default_indentation, 'indentation_command': indentation_command}) - elif re.search(r'\[RESTORE\]', line): - if len(formatting_stack) > 0: - prior_values = formatting_stack.pop() - spacing_command = prior_values['spacing_command'] - after_space = prior_values['after_space'] - default_indentation = prior_values['default_indentation'] - indentation_command = prior_values['indentation_command'] - elif re.search(r'\[TIGHTSPACING\]', line): - spacing_command = rtf_spacing['tight'] - default_spacing = 'tight' - after_space = after_space_multiplier * rtf_after_space[default_spacing] - default_indentation = False - elif re.search(r'\[SINGLESPACING\]', line): - spacing_command = rtf_spacing['single'] - default_spacing = 'single' - after_space = after_space_multiplier * rtf_after_space[default_spacing] - default_indentation = False - elif re.search(r'\[ONEANDAHALFSPACING\]', line): - spacing_command = rtf_spacing['oneandahalf'] - default_spacing = 'oneandahalf' - after_space = after_space_multiplier * rtf_after_space[default_spacing] - elif re.search(r'\[DOUBLESPACING\]', line): - spacing_command = rtf_spacing['double'] - default_spacing = 'double' - after_space = after_space_multiplier * rtf_after_space[default_spacing] - elif re.search(r'\[TRIPLESPACING\]', line): - spacing_command = rtf_spacing['triple'] - default_spacing = 'triple' - after_space = after_space_multiplier * rtf_after_space[default_spacing] - elif re.search(r'\[START_INDENTATION\]', line): - indentation_command = r'\\fi' + str(indentation_amount) + " " - elif re.search(r'\[STOP_INDENTATION\]', line): - indentation_command = r'\\fi0 ' - elif line != '': - special_after_space = None - special_spacing = None - if re.search(r'\[BORDER\]', line): - line = re.sub(r' *\[BORDER\] *', r'', line) - border_text = r'\\box \\brdrhair \\brdrw1 \\brdrcf1 \\brsp29 ' - else: - border_text = r'' - line = re.sub(r'{(\\pard\\intbl \\q[lrc] \\f[0-9]+ \\sa[0-9]+ \\li[0-9]+ \\fi[0-9]+.*?)\\par}', r'\1', line) - if re.search(r'\[NOPAR\]', line): - line = re.sub(r'{\\pard \\ql \\f[0-9]+ \\sa[0-9]+ \\li[0-9]+ \\fi-?[0-9]* *(.*?)\\par}', r'\1', line) - line = re.sub(r' *\[NOPAR\] *', r'', line) - n = re.search(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9\.]+ *[A-Za-z]+)\]', line) - m = re.search(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\]', line) - if n: - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r'\\ri-?[0-9]+ ', r'', line) - line = re.sub(r'\\li-?[0-9]+ ', r'\\li' + str(convert_length(n.group(1), 'twips')) + r' \\ri' + str(convert_length(n.group(2), 'twips')) + ' ', line) - line = re.sub(r'\[INDENTBY[^\]]*\]', '', line) - elif m: - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r'\\li-?[0-9]+ ', r'\\li' + str(convert_length(m.group(1), 'twips')) + ' ', line) - line = re.sub(r' *\[INDENTBY[^\]]*\] *', '', line) - elif re.search(r'\[NOINDENT\]', line): - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r' *\[NOINDENT\] *', '', line) - elif re.search(r'\[FLUSHLEFT\]', line): - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r' *\[FLUSHLEFT\] *', '', line) - special_after_space = after_space_multiplier * 1 - special_spacing = rtf_spacing['single'] - elif re.search(r'\[FLUSHRIGHT\]', line): - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r'\\ql', r'\\qr', line) - line = re.sub(r' *\[FLUSHRIGHT\] *', '', line) - special_after_space = after_space_multiplier * 1 - special_spacing = rtf_spacing['single'] - elif re.search(r'\[CENTER\]', line): - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r'\\ql', r'\\qc', line) - line = re.sub(r' *\[CENTER\] *', '', line) - elif re.search(r'\[BOLDCENTER\]', line): - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r'\\ql', r'\\qc \\b', line) - line = re.sub(r' *\[BOLDCENTER\] *', '', line) - elif indentation_command != '' and not re.search(r'\\widctlpar', line): - line = re.sub(r'\\fi-?[0-9]+ ', indentation_command, line) - if not re.search(r'\\s[0-9]', line): - if special_spacing: - spacing_command_to_use = special_spacing - else: - spacing_command_to_use = spacing_command - line = re.sub(r'\\pard ', r'\\pard ' + str(spacing_command_to_use) + str(border_text), line) - line = re.sub(r'\\pard\\intbl ', r'\\pard\\intbl ' + str(spacing_command_to_use) + str(border_text), line) - if not (re.search(r'\\fi0\\(endash|bullet)', line) or re.search(r'\\s[0-9]', line) or re.search(r'\\intbl', line)): - if special_after_space: - after_space_to_use = special_after_space - else: - after_space_to_use = after_space - if after_space_to_use > 0: - line = re.sub(r'\\sa[0-9]+ ', r'\\sa' + str(after_space_to_use) + ' ', line) - else: - line = re.sub(r'\\sa[0-9]+ ', r'\\sa0 ', line) - text += line + '\n' - text = re.sub(r'{\\pard \\sl[0-9]+\\slmult[0-9]+ \\ql \\f[0-9]+ \\sa[0-9]+ \\li[0-9]+ \\fi-?[0-9]*\s*\\par}', r'', text) - text = re.sub(r'\[MANUALSKIP\]', r'{\\pard \\sl0 \\ql \\f0 \\sa0 \\li0 \\fi0 \\par}', text) - return text - - -def docx_filter(text, metadata=None, question=None): - if metadata is None: - metadata = {} - text = text + "\n\n" - text = re.sub(r'\[\[([^\]]*)\]\]', r'\1', text) - text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_docx(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_include_docx(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_docx(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_include_docx(x, question=question), text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_include_docx, text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_include_docx, text) - text = re.sub(r'\[QR ([^\]]+)\]', qr_include_docx, text) - text = re.sub(r'\[MAP ([^\]]+)\]', '', text) - text = replace_fields(text) - # text = re.sub(r'\[FIELD ([^\]]+)\]', '', text) - text = re.sub(r'\[TARGET ([^\]]+)\]', '', text) - text = re.sub(r'\[YOUTUBE[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\[VIMEO[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\\clearpage *\\clearpage', '', text) - text = re.sub(r'\[START_INDENTATION\]', '', text) - text = re.sub(r'\[STOP_INDENTATION\]', '', text) - text = re.sub(r'\[BEGIN_CAPTION\](.+?)\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', '', text, flags=re.DOTALL) - text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\[BREAK\]\s*(.+?)\[END_TWOCOL\]', '', text, flags=re.DOTALL) - text = re.sub(r'\[TIGHTSPACING\] *', '', text) - text = re.sub(r'\[SINGLESPACING\] *', '', text) - text = re.sub(r'\[DOUBLESPACING\] *', '', text) - text = re.sub(r'\[ONEANDAHALFSPACING\] *', '', text) - text = re.sub(r'\[TRIPLESPACING\] *', '', text) - text = re.sub(r'\[NBSP\]', ' ', text) - text = re.sub(r'\[REDACTION_SPACE\]', "\u200B", text) - text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('ā–ˆ', x), text) - text = re.sub(r'\[ENDASH\]', '--', text) - text = re.sub(r'\[EMDASH\]', '---', text) - text = re.sub(r'\[HYPHEN\]', '-', text) - text = re.sub(r'\[CHECKBOX\]', '____', text) - text = re.sub(r'\[BLANK\]', r'__________________', text) - text = re.sub(r'\[BLANKFILL\]', r'__________________', text) - text = re.sub(r'\[PAGEBREAK\] *', '', text) - text = re.sub(r'\[PAGENUM\] *', '', text) - text = re.sub(r'\[TOTALPAGES\] *', '', text) - text = re.sub(r'\[SECTIONNUM\] *', '', text) - text = re.sub(r'\[SKIPLINE\] *', '\n\n', text) - text = re.sub(r'\[VERTICALSPACE\] *', '\n\n', text) - text = re.sub(r'\[NEWLINE\] *', '\n\n', text) - text = re.sub(r'\[NEWPAR\] *', '\n\n', text) - text = re.sub(r'\[BR\] *', '\n\n', text) - text = re.sub(r'\[TAB\] *', '', text) - text = re.sub(r' *\[END\] *', r'\n', text) - text = re.sub(r'\[BORDER\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[NOINDENT\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[FLUSHLEFT\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[FLUSHRIGHT\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[CENTER\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[BOLDCENTER\] *(.+?)\n *\n', r'**\1**\n\n', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\] *(.+?)\n *\n', r'\2', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9]+ *[A-Za-z]+)\] *(.+?)\n *\n', r'\3', text, flags=re.MULTILINE | re.DOTALL) - return text - - -def docx_template_filter(text, question=None, replace_newlines=True): - # logmessage('docx_template_filter') - if text == 'True': - return True - if text == 'False': - return False - if text == 'None': - return None - text = re.sub(r'\[\[([^\]]*)\]\]', r'\1', text) - text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_docx_template(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_include_docx_template(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_docx_template(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_include_docx_template(x, question=question), text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_include_docx_template, text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_include_docx_template, text) - text = re.sub(r'\[QR ([^\]]+)\]', qr_include_docx_template, text) - text = re.sub(r'\[MAP ([^\]]+)\]', '', text) - text = replace_fields(text) - # text = re.sub(r'\[FIELD ([^\]]+)\]', '', text) - text = re.sub(r'\[TARGET ([^\]]+)\]', '', text) - text = re.sub(r'\[YOUTUBE[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\[VIMEO[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\\clearpage *\\clearpage', '', text) - text = re.sub(r'\[START_INDENTATION\]', '', text) - text = re.sub(r'\[STOP_INDENTATION\]', '', text) - text = re.sub(r'\[BEGIN_CAPTION\](.+?)\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', '', text, flags=re.DOTALL) - text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\[BREAK\]\s*(.+?)\[END_TWOCOL\]', '', text, flags=re.DOTALL) - text = re.sub(r'\[TIGHTSPACING\] *', '', text) - text = re.sub(r'\[SINGLESPACING\] *', '', text) - text = re.sub(r'\[DOUBLESPACING\] *', '', text) - text = re.sub(r'\[ONEANDAHALFSPACING\] *', '', text) - text = re.sub(r'\[TRIPLESPACING\] *', '', text) - text = re.sub(r'\[NBSP\]', ' ', text) - text = re.sub(r'\[REDACTION_SPACE\]', "\u200B", text) - # text = re.sub(r'\[REDACTION_SPACE\]', r'', text) - text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('ā–ˆ', x), text) - # text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('X', x), text) - text = re.sub(r'\[ENDASH\]', '--', text) - text = re.sub(r'\[EMDASH\]', '---', text) - text = re.sub(r'\[HYPHEN\]', '-', text) - text = re.sub(r'\\', '', text) - text = re.sub(r'\[CHECKBOX\]', '____', text) - text = re.sub(r'\[BLANK\]', r'__________________', text) - text = re.sub(r'\[BLANKFILL\]', r'__________________', text) - text = re.sub(r'\[PAGEBREAK\] *', '', text) - text = re.sub(r'\[PAGENUM\] *', '', text) - text = re.sub(r'\[TOTALPAGES\] *', '', text) - text = re.sub(r'\[SECTIONNUM\] *', '', text) - text = re.sub(r'\[SKIPLINE\] *', '', text) - text = re.sub(r'\[VERTICALSPACE\] *', '', text) - text = re.sub(r'\[NEWLINE\] *', '', text) - # text = re.sub(r'\n *\n', '[NEWPAR]', text) - if replace_newlines: - text = re.sub(r'\n', ' ', text) - text = re.sub(r'\[NEWPAR\] *', '', text) - text = re.sub(r'\[TAB\] *', '\t', text) - text = re.sub(r'\[NEWPAR\]', '', text) - text = re.sub(r' *\[END\] *', r'', text) - text = re.sub(r'\[BORDER\] *', r'', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[NOINDENT\] *', r'', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[FLUSHLEFT\] *', r'', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[FLUSHRIGHT\] *', r'', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[CENTER\] *', r'', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[BOLDCENTER\] *', r'', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\] *(.+?)\n *\n', r'\2', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9]+ *[A-Za-z]+)\] *(.+?)\n *\n', r'\3', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[BR\]', '', text) - text = re.sub(r'\[SKIPLINE\]', '', text) - text = re.sub(r'{([{%#])', '{' + zerowidth + r'\1', re.sub(r'([}%#])}', r'\1' + zerowidth + '}', text)) - return text - - -def metadata_filter(text, doc_format): - if doc_format == 'pdf': - text = re.sub(r'\*\*([^\*]+?)\*\*', r'\\begingroup\\bfseries \1\\endgroup {}', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\*([^\*]+?)\*', r'\\begingroup\\itshape \1\\endgroup {}', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\_\_([^\_]+?)\_\_', r'\\begingroup\\bfseries \1\\endgroup {}', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\_([^\_]+?)\_*', r'\\begingroup\\itshape \1\\endgroup {}', text, flags=re.MULTILINE | re.DOTALL) - return text - - -def redact_latex(match): - return '\\redactword{' + str(escape_latex(match.group(1))) + '}' - - -def pdf_filter(text, metadata=None, question=None): - if metadata is None: - metadata = {} - text = text + "\n\n" - text = re.sub(r'\[\[([^\]]*)\]\]', r'\1', text) - text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_string(x, emoji=True, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_include_string(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_string(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_include_string(x, question=question), text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_include_string, text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_include_string, text) - text = re.sub(r'\[QR ([^\]]+)\]', qr_include_string, text) - text = re.sub(r'\[MAP ([^\]]+)\]', '', text) - text = replace_fields(text) - # text = re.sub(r'\[FIELD ([^\]]+)\]', '', text) - text = re.sub(r'\[TARGET ([^\]]+)\]', '', text) - text = re.sub(r'\[YOUTUBE[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\[VIMEO[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\$\$+', '$', text) - text = re.sub(r'\\clearpage *\\clearpage', r'\\clearpage', text) - text = re.sub(r'\[BORDER\]\s*\[(BEGIN_TWOCOL|BEGIN_CAPTION|TIGHTSPACING|SINGLESPACING|DOUBLESPACING|START_INDENTATION|STOP_INDENTATION|NOINDENT|FLUSHLEFT|FLUSHRIGHT|CENTER|BOLDCENTER|INDENTBY[^\]]*)\]', r'[\1] [BORDER]', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[START_INDENTATION\]', r'\\setlength{\\parindent}{\\myindentamount}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[STOP_INDENTATION\]', r'\\setlength{\\parindent}{0in}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[BEGIN_CAPTION\](.+?)\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', pdf_caption, text, flags=re.DOTALL) - text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\[BREAK\]\s*(.+?)\[END_TWOCOL\]', pdf_two_col, text, flags=re.DOTALL) - text = re.sub(r'\[TIGHTSPACING\]\s*', r'\\singlespacing\\setlength{\\parskip}{0pt}\\setlength{\\parindent}{0pt}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[SINGLESPACING\]\s*', r'\\singlespacing\\setlength{\\parskip}{\\myfontsize}\\setlength{\\parindent}{0pt}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[DOUBLESPACING\]\s*', r'\\doublespacing\\setlength{\\parindent}{\\myindentamount}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[ONEANDAHALFSPACING\]\s*', r'\\onehalfspacing\\setlength{\\parindent}{\\myindentamount}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[TRIPLESPACING\]\s*', r'\\setlength{\\parindent}{\\myindentamount}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[NBSP\]', r'\\myshow{\\nonbreakingspace}', text) - text = re.sub(r'\[REDACTION_SPACE\]', r'\\redactword{~}\\hspace{0pt}', text) - text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', redact_latex, text) - text = re.sub(r'\[ENDASH\]', r'\\myshow{\\myendash}', text) - text = re.sub(r'\[EMDASH\]', r'\\myshow{\\myemdash}', text) - text = re.sub(r'\[HYPHEN\]', r'\\myshow{\\myhyphen}', text) - text = re.sub(r'\[CHECKBOX\]', r'{\\rule{0.3in}{0.4pt}}', text) - text = re.sub(r'\[BLANK\]', r'\\leavevmode{\\xrfill[-2pt]{0.4pt}}', text) - text = re.sub(r'\[BLANKFILL\]', r'\\leavevmode{\\xrfill[-2pt]{0.4pt}}', text) - text = re.sub(r'\[PAGEBREAK\]\s*', r'\\clearpage ', text) - text = re.sub(r'\[PAGENUM\]', r'\\myshow{\\thepage\\myxspace}', text) - text = re.sub(r'\[TOTALPAGES\]', r'\\myshow{\\pageref*{LastPage}\\myxspace}', text) - text = re.sub(r'\[SECTIONNUM\]', r'\\myshow{\\thesection\\myxspace}', text) - text = re.sub(r'\[VERTICALSPACE\] *', r'\\rule[-24pt]{0pt}{0pt}', text) - text = re.sub(r'\[NEWLINE\] *', r'\\newline ', text) - text = re.sub(r'\[NEWPAR\] *', r'\\par ', text) - text = re.sub(r'\[BR\] *', r'\\manuallinebreak ', text) - text = re.sub(r'\[TAB\] *', r'\\manualindent ', text) - text = re.sub(r' *\[END\] *', r'\n', text) - text = re.sub(r'\[NOINDENT\] *', r'\\noindent ', text) - text = re.sub(r'\[FLUSHLEFT\] *(.+?)\n *\n', flushleft_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[FLUSHRIGHT\] *(.+?)\n *\n', flushright_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[CENTER\] *(.+?)\n *\n', center_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[BOLDCENTER\] *(.+?)\n *\n', boldcenter_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\] *(.+?)\n *\n', indentby_left_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9]+ *[A-Za-z]+)\] *(.+?)\n *\n', indentby_both_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[BORDER\] *(.+?)\n *\n', border_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\s*\[SKIPLINE\]\s*', r'\\par\\myskipline ', text) - return text - - -def html_filter(text, status=None, question=None, embedder=None, default_image_width=None, external=False): - if question is None and status is not None: - question = status.question - text = text + "\n\n" - text = re.sub(r'^[|] (.*)$', r'\1
', text, flags=re.MULTILINE) - text = replace_fields(text, status=status, embedder=embedder) - # if embedder is not None: - # text = re.sub(r'\[FIELD ([^\]]+)\]', lambda x: embedder(status, x.group(1)), text) - # else: - # text = re.sub(r'\[FIELD ([^\]]+)\]', 'ERROR: FIELD cannot be used here', text) - text = re.sub(r'\[TARGET ([^\]]+)\]', target_html, text) - if docassemble.base.functions.this_thread.evaluation_context != 'docx': - text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_url_string(x, emoji=True, question=question, external=external, status=status), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_url_string(x, question=question, external=external, status=status), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_url_string(x, question=question, external=external, status=status), text) - text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_url_string(x, question=question, default_image_width=default_image_width, external=external, status=status), text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_url_string, text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_url_string, text) - text = re.sub(r'\[QR ([^,\]]+)\]', qr_url_string, text) - if map_match.search(text): - text = map_match.sub((lambda x: map_string(x.group(1), status)), text) - # width="420" height="315" - text = re.sub(r'\[YOUTUBE ([^\]]+)\]', r'
', text) - text = re.sub(r'\[YOUTUBE4:3 ([^\]]+)\]', r'
', text) - text = re.sub(r'\[YOUTUBE16:9 ([^\]]+)\]', r'
', text) - # width="500" height="281" - text = re.sub(r'\[VIMEO ([^\]]+)\]', r'
', text) - text = re.sub(r'\[VIMEO4:3 ([^\]]+)\]', r'
', text) - text = re.sub(r'\[VIMEO16:9 ([^\]]+)\]', r'
', text) - text = re.sub(r'\[BEGIN_CAPTION\](.+?)\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', html_caption, text, flags=re.DOTALL) - text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\[BREAK\]\s*(.+?)\[END_TWOCOL\]', html_two_col, text, flags=re.DOTALL) - text = re.sub(r'\[NBSP\]', r' ', text) - text = re.sub(r'\[REDACTION_SPACE\]', '█​', text) - text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('█', x), text) - text = re.sub(r'\[ENDASH\]', r'–', text) - text = re.sub(r'\[EMDASH\]', r'—', text) - text = re.sub(r'\[HYPHEN\]', r'-', text) - text = re.sub(r'\[CHECKBOX\]', r'    ', text) - text = re.sub(r'\[BLANK\]', r'            ', text) - text = re.sub(r'\[BLANKFILL\]', r'                              ', text) - text = re.sub(r'\[PAGEBREAK\] *', r'', text) - text = re.sub(r'\[PAGENUM\] *', r'', text) - text = re.sub(r'\[SECTIONNUM\] *', r'', text) - text = re.sub(r'\[SKIPLINE\] *', r'
', text) - text = re.sub(r'\[NEWLINE\] *', r'
', text) - text = re.sub(r'\[NEWPAR\] *', r'

', text) - text = re.sub(r'\[BR\] *', r'
', text) - text = re.sub(r'\[TAB\] *', '', text) - text = re.sub(r' *\[END\] *', r'\n', text) - lines = re.split(r'\n *\n', text) - text = '' - spacing_class = None - doing_indentation = False - for line in lines: - classes = set() - styles = {} - if re.search(r'\[TIGHTSPACING\]', line): - spacing_class = 'daspacingtight' - if re.search(r'\[SINGLESPACING\]', line): - spacing_class = 'daspacingsingle' - if re.search(r'\[DOUBLESPACING\]', line): - spacing_class = 'daspacingdouble' - if re.search(r'\[ONEANDAHALFSPACING\]', line): - spacing_class = 'daspacingoneandahalf' - if re.search(r'\[TRIPLESPACING\]', line): - spacing_class = 'daspacingtriple' - if re.search(r'\[START_INDENTATION\]', line): - doing_indentation = True - if re.search(r'\[STOP_INDENTATION\]', line): - doing_indentation = False - if spacing_class: - classes.add(spacing_class) - if doing_indentation and not re.search(r'\[NOINDENT\]', line): - styles['text-indent'] = '36px' - if re.search(r'\[BORDER\]', line): - classes.add('daborder') - if re.search(r'\[FLUSHLEFT\]', line): - classes.add('daflushleft') - if re.search(r'\[FLUSHRIGHT\]', line): - classes.add('daflushright') - if re.search(r'\[CENTER\]', line): - classes.add('dacenter') - if re.search(r'\[BOLDCENTER\]', line): - classes.add('dacenter') - classes.add('dabold') - m = re.search(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\]', line) - if m: - styles["padding-left"] = str(convert_length(m.group(1), 'px')) + 'px' - m = re.search(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9]+ *[A-Za-z]+)\]', line) - if m: - styles["margin-left"] = str(convert_length(m.group(1), 'px')) + 'px' - styles["margin-right"] = str(convert_length(m.group(2), 'px')) + 'px' - orig_length = len(line) - line = re.sub(r'\[(BORDER|NOINDENT|FLUSHLEFT|FLUSHRIGHT|BOLDCENTER|CENTER|TIGHTSPACING|SINGLESPACING|DOUBLESPACING|ONEANDAHALFSPACING|TRIPLESPACING|START_INDENTATION|STOP_INDENTATION)\] *', r'', line) - line = re.sub(r'\[INDENTBY[^\]]*\] *', r'', line) - if orig_length > 0 and len(line) == 0: - continue - if line.startswith('>'): - line = re.sub(r'^> *', '', line) - text += "> " - if len(classes) > 0 or len(styles) > 0: - text += ' 0: - text += ' style="' + "".join(map(lambda x: str(x[0]) + ":" + x[1] + ';', styles.items())) + '"' - text += '>' - text += line + '\n\n' - text = re.sub(r'\n+$', r'', text) - return text - - -def clean_markdown_to_latex(string): - string = re.sub(r'\s*\[SKIPLINE\]\s*', r'\\par\\myskipline ', string) - string = re.sub(r'^[\n ]+', '', string) - string = re.sub(r'[\n ]+$', '', string) - string = re.sub(r' *\n *$', '\n', string) - string = re.sub(r'\n{2,}', '[NEWLINE]', string) - string = re.sub(r'\[BR\]', '[NEWLINE]', string) - string = re.sub(r'\[(NOINDENT|FLUSHLEFT|FLUSHRIGHT|CENTER|BOLDCENTER|TIGHTSPACING|SINGLESPACING|DOUBLESPACING|START_INDENTATION|STOP_INDENTATION|PAGEBREAK)\]\s*', '', string) - string = re.sub(r'\*\*([^\*]+?)\*\*', r'\\textbf{\1}', string) - string = re.sub(r'\*([^\*]+?)\*', r'\\emph{\1}', string) - string = re.sub(r'(?' - - -def target_html(match): - target = match.group(1) - target = re.sub(r'[^A-Za-z0-9\_]', r'', str(target)) - return '' - - -def pdf_two_col(match, add_line=False): - firstcol = clean_markdown_to_latex(match.group(1)) - secondcol = clean_markdown_to_latex(match.group(2)) - if add_line: - return '\\noindent\\begingroup\\singlespacing\\setlength{\\parskip}{0pt}\\mynoindent\\begin{tabular}{@{}m{0.49\\textwidth}|@{\\hspace{1em}}m{0.49\\textwidth}@{}}{' + firstcol + '} & {' + secondcol + '} \\\\ \\end{tabular}\\endgroup\\myskipline' - return '\\noindent\\begingroup\\singlespacing\\setlength{\\parskip}{0pt}\\mynoindent\\begin{tabular}{@{}m{0.49\\textwidth}@{\\hspace{1em}}m{0.49\\textwidth}@{}}{' + firstcol + '} & {' + secondcol + '} \\\\ \\end{tabular}\\endgroup\\myskipline' - - -def html_caption(match): - firstcol = match.group(1) - secondcol = match.group(2) - firstcol = re.sub(r'^\s+', '', firstcol) - firstcol = re.sub(r'\s+$', '', firstcol) - secondcol = re.sub(r'^\s+', '', secondcol) - secondcol = re.sub(r'\s+$', '', secondcol) - firstcol = markdown_to_html(firstcol) - secondcol = markdown_to_html(secondcol) - return '
' + firstcol + '' + secondcol + '
' - - -def html_two_col(match): - firstcol = markdown_to_html(match.group(1)) - secondcol = markdown_to_html(match.group(2)) - return '
' + firstcol + '' + secondcol + '
' - - -def pdf_caption(match): - return pdf_two_col(match, add_line=False) - - -def add_newlines(string): - string = re.sub(r'\[(BR)\]', r'[NEWLINE]', string) - string = re.sub(r' *\n', r'\n', string) - string = re.sub(r'(? 0 and file_info['width'] > 0: - scale = float(pixels)/float(file_info['width']) - # logmessage("scale is " + str(scale)) - if scale*float(file_info['height']) > float(MAX_HEIGHT_POINTS): - scale = float(MAX_HEIGHT_POINTS)/float(file_info['height']) - # logmessage("scale is " + str(scale)) - if scale*float(file_info['width']) > float(MAX_WIDTH_POINTS): - scale = float(MAX_WIDTH_POINTS)/float(file_info['width']) - # logmessage("scale is " + str(scale)) - # scale *= 100.0 - # logmessage("scale is " + str(scale)) - # scale = int(scale) - # logmessage("scale is " + str(scale)) - wtwips = int(scale*float(file_info['width'])*20.0) - htwips = int(scale*float(file_info['height'])*20.0) - image = Image(file_info['fullpath']) - image.Data = re.sub(r'\\picwgoal([0-9]+)', r'\\picwgoal' + str(wtwips), image.Data) - image.Data = re.sub(r'\\pichgoal([0-9]+)', r'\\pichgoal' + str(htwips), image.Data) - else: - image = Image(file_info['fullpath']) - if insert_page_breaks: - content = '\\page ' - else: - content = '' - # logmessage(content + image.Data) - return content + image.Data - -unit_multipliers = {'twips': 0.0500, 'hp': 0.5, 'in': 72, 'pt': 1, 'px': 1, 'em': 12, 'cm': 28.346472} - - -def convert_length(length, unit): - value = pixels_in(length) - if unit in unit_multipliers: - size = float(value)/float(unit_multipliers[unit]) - return int(size) - logmessage("Unit " + str(unit) + " is not a valid unit") - return 300 - - -def pixels_in(length): - m = re.search(r"([0-9.]+) *([a-z]+)", str(length).lower()) - if m: - value = float(m.group(1)) - unit = m.group(2) - # logmessage("value is " + str(value) + " and unit is " + unit) - if unit in unit_multipliers: - size = float(unit_multipliers[unit]) * value - # logmessage("size is " + str(size)) - return int(size) - logmessage("Could not read " + str(length)) - return 300 - - -def image_url_string(match, emoji=False, question=None, default_image_width=None, external=False, status=None): - file_reference = match.group(1) - try: - width = match.group(2) - assert width != 'None' - except: - if default_image_width is not None: - width = default_image_width - else: - width = "300px" - if width == "full": - width = "300px" - if match.lastindex == 3: - if match.group(3) != 'None': - alt_text = 'alt=' + json.dumps(match.group(3)) + ' ' - else: - alt_text = '' - else: - alt_text = '' - return image_url(file_reference, alt_text, width, emoji=emoji, question=question, external=external, status=status) - - -def image_url(file_reference, alt_text, width, emoji=False, question=None, external=False, status=None): - if question and file_reference in question.interview.images: - if status and question.interview.images[file_reference].attribution is not None: - status.attributions.add(question.interview.images[file_reference].attribution) - file_reference = question.interview.images[file_reference].get_reference() - file_info = server.file_finder(file_reference, question=question) - if 'mimetype' in file_info and file_info['mimetype']: - if re.search(r'^audio', file_info['mimetype']): - urls = get_audio_urls([{'text': "[FILE " + file_reference + "]", 'package': None, 'type': 'audio'}], question=question) - if len(urls) > 0: - return audio_control(urls) - return '' - if re.search(r'^video', file_info['mimetype']): - urls = get_video_urls([{'text': "[FILE " + file_reference + "]", 'package': None, 'type': 'video'}], question=question) - if len(urls) > 0: - return video_control(urls) - return '' - if 'extension' in file_info and file_info['extension'] is not None: - if re.match(r'.*%$', width): - width_string = "width:" + width - stack_width_string = width_string - else: - width_string = "max-width:" + width - stack_width_string = "width:" + width - if emoji: - width_string += ';vertical-align: middle' - alt_text = 'alt="" ' - the_url = server.url_finder(file_reference, _question=question, display_filename=file_info['filename'], _external=external) - if the_url is None: - return '[ERROR: File reference ' + str(file_reference) + ' cannot be displayed]' - if width_string == 'width:100%': - extra_class = ' dawideimage' - else: - extra_class = '' - if file_info.get('extension', '') in ('png', 'jpg', 'gif', 'svg', 'jpe', 'jpeg'): - try: - if file_info.get('extension', '') == 'svg': - attributes = ET.parse(file_info['fullpath']).getroot().attrib - layout_width = attributes['width'] - layout_height = attributes['height'] - else: - with PIL.Image.open(file_info['fullpath']) as im: - layout_width, layout_height = im.size - return '' - except: - return '' - if file_info['extension'] in ('pdf', 'docx', 'rtf', 'doc', 'odt'): - if file_info['extension'] in ('docx', 'rtf', 'doc', 'odt') and not os.path.isfile(file_info['path'] + '.pdf'): - server.fg_make_pdf_for_word_path(file_info['path'], file_info['extension']) - server.fg_make_png_for_pdf_path(file_info['path'] + ".pdf", 'screen', page=1) - if re.match(r'[0-9]+', str(file_reference)): - sf = server.SavedFile(int(file_reference), fix=True) - sf.finalize() - if 'pages' not in file_info: - try: - with Pdf.open(file_info['path'] + '.pdf') as reader: - file_info['pages'] = len(reader.pages) - except: - file_info['pages'] = 1 - the_image_url = server.url_finder(file_reference, size="screen", page=1, _question=question, _external=external) - if the_image_url is None: - return '[ERROR: File reference ' + str(file_reference) + ' cannot be displayed]' - if 'filename' in file_info: - title = ' title="' + file_info['filename'] - if 'pages' in file_info and file_info['pages'] > 1: - title += " (" + str(file_info['pages']) + " " + word('pages') + ")" - title += '"' - else: - if 'pages' in file_info and file_info['pages'] > 1: - title = ' title="' + str(file_info['pages']) + " " + word('pages') + '"' - else: - title = '' - if alt_text == '': - the_alt_text = 'alt=' + json.dumps(word("Thumbnail image of document")) + ' ' - else: - the_alt_text = alt_text - try: - with Pdf.open(file_info['path'] + '.pdf') as reader: - layout_width = reader.pages[0].mediabox[2] - reader.pages[0].mediabox[0] - layout_height = reader.pages[0].mediabox[3] - reader.pages[0].mediabox[1] - if width_string == 'width:100%': - output = '' - else: - if 'pages' in file_info and file_info['pages'] >= 1: - extra_pages = min(2, file_info['pages'] - 1) - else: - extra_pages = 2 - aspect_ratio = 1.0*layout_width/layout_height - stack_width_string += "; height: auto; aspect-ratio: " + str(aspect_ratio) + ";" - output = '
' + (('
') * extra_pages) + '
' - except: - output = '' - return output - return '' + file_info['filename'] + '' - return '[Invalid image reference; reference=' + str(file_reference) + ', width=' + str(width) + ', filename=' + file_info.get('filename', 'unknown') + ']' - - -def qr_url_string(match): - string = match.group(1) - try: - width = match.group(2) - assert width != 'None' - except: - width = "300px" - if width == "full": - width = "300px" - if match.lastindex == 3: - if match.group(3) != 'None': - alt_text = str(match.group(3)) - else: - alt_text = word(f"A QR code that goes to {string}") - else: - alt_text = word(f"A QR code that goes to {string}") - width_string = "width:" + width - im = qrcode.make(string, image_factory=qrcode.image.svg.SvgPathFillImage) - output = BytesIO() - im.save(output) - the_image = output.getvalue().decode() - the_image = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n", '', the_image) - the_image = re.sub(r'height="[0-9]+mm" ', '', the_image) - the_image = re.sub(r'width="[0-9]+mm" ', '', the_image) - m = re.search(r'(viewBox="[^"]+")', the_image) - if m: - viewbox = m.group(1) - else: - viewbox = "" - return '' + the_image + '' + alt_text + '' - - -def convert_pixels(match): - pixels = match.group(1) - return str(int(pixels)/72.0) + "in" - - -def convert_percent(match): - percentage = match.group(1) - return str(float(percentage)/100.0) + '\\textwidth' - - -def image_include_string(match, emoji=False, question=None): - file_reference = match.group(1) - if question and file_reference in question.interview.images: - file_reference = question.interview.images[file_reference].get_reference() - try: - width = match.group(2) - assert width != 'None' - width = re.sub(r'^(.*)px', convert_pixels, width) - width = re.sub(r'^(.*)%', convert_percent, width) - if width == "full": - width = '\\textwidth' - except: - width = DEFAULT_IMAGE_WIDTH - if match.lastindex == 3: - alt_text = match.group(3) - else: - alt_text = None - file_info = server.file_finder(file_reference, question=question) - if 'path' in file_info and 'extension' in file_info: - convert_svg_to_eps(file_info) - if file_info['extension'] == 'gif': - with tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".png", delete=False) as png_file: - try: - with PIL.Image.open(file_info['fullpath']) as im: - im.save(png_file.name) - png_file.close() - file_info['path'] = png_file.name - file_info['fullpath'] = png_file.name - file_info['extension'] = 'png' - file_info['mimetype'] = 'image/png' - except BaseException as err: - logmessage("Could not convert GIF to PNG: " + err.__class__.__name__ + ": " + str(err)) - if 'mimetype' in file_info and file_info['mimetype']: - if re.search(r'^(audio|video)', file_info['mimetype']): - return '[reference to file type that cannot be displayed]' - if 'path' in file_info: - if 'extension' in file_info: - if file_info['extension'] in ['png', 'jpg', 'pdf', 'eps', 'jpe', 'jpeg', 'docx', 'rtf', 'doc', 'odt']: - if file_info['extension'] == 'pdf': - output = '\\includepdf[pages={-}]{' + file_info['path'] + '.pdf}' - elif file_info['extension'] in ('docx', 'rtf', 'doc', 'odt'): - if not os.path.isfile(file_info['path'] + '.pdf'): - server.fg_make_pdf_for_word_path(file_info['path'], file_info['extension']) - output = '\\includepdf[pages={-}]{' + file_info['path'] + '.pdf}' - else: - if alt_text: - alt_text_string = ', alt={' + re.sub(r'[{}]', '', alt_text) + '}' - else: - alt_text_string = '' - if emoji: - output = '\\raisebox{-.6\\dp\\strutbox}{\\mbox{\\includegraphics[width=' + width + alt_text_string + ']{' + file_info['path'] + '}}}' - else: - output = '\\mbox{\\includegraphics[width=' + width + alt_text_string + ']{' + file_info['path'] + '}}' - if width == '\\textwidth': - output = '\\clearpage ' + output + '\\clearpage ' - return output - return '[invalid graphics reference]' - - -def image_include_docx(match, question=None): - file_reference = match.group(1) - if question and file_reference in question.interview.images: - file_reference = question.interview.images[file_reference].get_reference() - try: - width = match.group(2) - assert width != 'None' - width = re.sub(r'^(.*)px', convert_pixels, width) - if width == "full": - width = '100%' - except: - width = DEFAULT_IMAGE_WIDTH - if match.lastindex == 3: - alt_text = match.group(3) - else: - alt_text = None - if not alt_text: - alt_text = '' - file_info = server.file_finder(file_reference, question=question) - if 'mimetype' in file_info and file_info['mimetype']: - if re.search(r'^(audio|video)', file_info['mimetype']): - return '[reference to file type that cannot be displayed]' - if 'path' in file_info: - if 'extension' in file_info: - convert_svg_to_eps(file_info) - if file_info['extension'] in ('docx', 'rtf', 'doc', 'odt'): - if not os.path.isfile(file_info['path'] + '.pdf'): - server.fg_make_pdf_for_word_path(file_info['path'], file_info['extension']) - output = '![' + alt_text + '](' + file_info['path'] + '.pdf){width=' + width + '}' - return output - if file_info['extension'] in ['png', 'jpg', 'gif', 'pdf', 'eps', 'jpe', 'jpeg']: - output = '![' + alt_text + '](' + file_info['fullpath'] + '){width=' + width + '}' - return output - return '[invalid graphics reference]' - - -def qr_include_string(match): - string = match.group(1) - try: - width = match.group(2) - assert width != 'None' - width = re.sub(r'^(.*)px', convert_pixels, width) - if width == "full": - width = '\\textwidth' - except: - width = DEFAULT_IMAGE_WIDTH - if match.lastindex == 3: - alt_text = match.group(3) - else: - alt_text = None - im = qrcode.make(string) - with tempfile.NamedTemporaryFile(prefix="datemp", suffix=".png", delete=False) as the_image: - # docassemble.base.functions.this_thread.temporary_resources.add(the_image.name) - im.save(the_image.name) - if alt_text: - alt_text_string = ', alt={' + re.sub(r'[{}]', '', alt_text) + '}' - else: - alt_text_string = '' - output = '\\mbox{\\includegraphics[width=' + width + alt_text_string + ']{' + the_image.name + '}}' - if width == '\\textwidth': - output = '\\clearpage ' + output + '\\clearpage ' - # logmessage("Output is " + output) - return output - - -def qr_include_docx(match): - string = match.group(1) - try: - width = match.group(2) - assert width != 'None' - width = re.sub(r'^(.*)px', convert_pixels, width) - if width == "full": - width = '100%' - except: - width = DEFAULT_IMAGE_WIDTH - if match.lastindex == 3: - alt_text = match.group(3) - else: - alt_text = None - if not alt_text: - alt_text = '' - im = qrcode.make(string) - with tempfile.NamedTemporaryFile(prefix="datemp", suffix=".png", delete=False) as the_image: - # docassemble.base.functions.this_thread.temporary_resources.add(the_image.name) - im.save(the_image.name) - output = '![' + alt_text + '](' + the_image.name + '){width=' + width + '}' - return output - - -def rtf_caption_table(match): - table_text = """\\trowd \\irow0\\irowband0\\lastrow \\ltrrow\\ts24\\trgaph108\\trleft0\\trbrdrt\\brdrs\\brdrw10 \\trbrdrl\\brdrs\\brdrw10 \\trbrdrb\\brdrs\\brdrw10 \\trbrdrr\\brdrs\\brdrw10 \\trbrdrh\\brdrs\\brdrw10 \\trbrdrv\\brdrs\\brdrw10 -\\trftsWidth1\\trftsWidthB3\\trftsWidthA3\\trautofit1\\trpaddl108\\trpaddr108\\trpaddfl3\\trpaddft3\\trpaddfb3\\trpaddfr3\\trcbpat1\\trcfpat1\\tblrsid1508006\\tbllkhdrrows\\tbllkhdrcols\\tbllknocolband\\tblind0\\tblindtype3 \\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone -\\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrs\\brdrw10 \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx4680\\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrs\\brdrw10 \\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx9468\\pard\\plain \\ltrpar -\\ql \\li0\\ri0\\widctlpar\\intbl\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0\\pararsid1508006\\yts24 \\rtlch\\fcs1 \\af0\\afs22\\alang1025 \\ltrch\\fcs0 \\fs22\\lang1033\\langfe1033\\cgrid\\langnp1033\\langfenp1033 { [SAVE][TIGHTSPACING][STOP_INDENTATION]""" + match.group(1) + """}{\\cell}{""" + match.group(2) + """[RESTORE]}{\\cell}\\pard\\plain \\ltrpar -\\ql \\li0\\ri0\\sa200\\sl276\\slmult1\\widctlpar\\intbl\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0 \\rtlch\\fcs1 \\af0\\afs22\\alang1025 \\ltrch\\fcs0 \\fs24\\lang1033\\langfe1033\\cgrid\\langnp1033\\langfenp1033 {\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\insrsid10753242 -\\trowd \\irow0\\irowband0\\lastrow \\ltrrow\\ts24\\trgaph108\\trleft0\\trbrdrt\\brdrs\\brdrw10 \\trbrdrl\\brdrs\\brdrw10 \\trbrdrb\\brdrs\\brdrw10 \\trbrdrr\\brdrs\\brdrw10 \\trbrdrh\\brdrs\\brdrw10 \\trbrdrv\\brdrs\\brdrw10 -\\trftsWidth1\\trftsWidthB3\\trftsWidthA3\\trautofit1\\trpaddl108\\trpaddr108\\trpaddfl3\\trpaddft3\\trpaddfb3\\trpaddfr3\\trcbpat1\\trcfpat1\\tblrsid1508006\\tbllkhdrrows\\tbllkhdrcols\\tbllknocolband\\tblind0\\tblindtype3 \\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone -\\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrs\\brdrw10 \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx4680\\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrs\\brdrw10 \\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx9468\\row }""" - table_text += """\\pard \\ltrpar -\\qc \\li0\\ri0\\sb0\\sl240\\slmult1\\widctlpar\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0\\itap0\\pararsid10753242""" - table_text = re.sub(r'\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0', r'\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\sl240 \\slmult1', table_text) - return table_text + '[MANUALSKIP]' - - -def rtf_two_col(match): - table_text = """\\trowd \\irow0\\irowband0\\lastrow \\ltrrow\\ts24\\trgaph108\\trleft0\\trbrdrt\\brdrs\\brdrw10 \\trbrdrl\\brdrs\\brdrw10 \\trbrdrb\\brdrs\\brdrw10 \\trbrdrr\\brdrs\\brdrw10 \\trbrdrh\\brdrs\\brdrw10 \\trbrdrv\\brdrs\\brdrw10 -\\trftsWidth1\\trftsWidthB3\\trftsWidthA3\\trautofit1\\trpaddl108\\trpaddr108\\trpaddfl3\\trpaddft3\\trpaddfb3\\trpaddfr3\\trcbpat1\\trcfpat1\\tblrsid1508006\\tbllkhdrrows\\tbllkhdrcols\\tbllknocolband\\tblind0\\tblindtype3 \\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone -\\clbrdrb\\brdrnone \\clbrdrr\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx4680\\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone \\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx9468\\pard\\plain \\ltrpar -\\ql \\li0\\ri0\\widctlpar\\intbl\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0\\pararsid1508006\\yts24 \\rtlch\\fcs1 \\af0\\afs22\\alang1025 \\ltrch\\fcs0 \\fs22\\lang1033\\langfe1033\\cgrid\\langnp1033\\langfenp1033 {\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\insrsid2427490 [SAVE][TIGHTSPACING][STOP_INDENTATION]""" + match.group(1) + """}{\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\insrsid10753242\\charrsid2427490 \\cell}{""" + match.group(2) + """[RESTORE]}{\\cell}\\pard\\plain \\ltrpar -\\ql \\li0\\ri0\\sa200\\sl276\\slmult1\\widctlpar\\intbl\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0 \\rtlch\\fcs1 \\af0\\afs22\\alang1025 \\ltrch\\fcs0 \\fs24\\lang1033\\langfe1033\\cgrid\\langnp1033\\langfenp1033 {\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\insrsid10753242 -\\trowd \\irow0\\irowband0\\lastrow \\ltrrow\\ts24\\trgaph108\\trleft0\\trbrdrt\\brdrs\\brdrw10 \\trbrdrl\\brdrs\\brdrw10 \\trbrdrb\\brdrs\\brdrw10 \\trbrdrr\\brdrs\\brdrw10 \\trbrdrh\\brdrs\\brdrw10 \\trbrdrv\\brdrs\\brdrw10 -\\trftsWidth1\\trftsWidthB3\\trftsWidthA3\\trautofit1\\trpaddl108\\trpaddr108\\trpaddfl3\\trpaddft3\\trpaddfb3\\trpaddfr3\\trcbpat1\\trcfpat1\\tblrsid1508006\\tbllkhdrrows\\tbllkhdrcols\\tbllknocolband\\tblind0\\tblindtype3 \\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone -\\clbrdrb\\brdrnone \\clbrdrr\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx4680\\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone \\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx9468\\row }""" - table_text += """\\pard \\ltrpar -\\qc \\li0\\ri0\\sb0\\sl240\\slmult1\\widctlpar\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0\\itap0\\pararsid10753242""" - table_text = re.sub(r'\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0', r'\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\sl240 \\slmult1', table_text) - return table_text + '[MANUALSKIP]' - - -def get_icon_html(text): - icons_setting = docassemble.base.functions.get_config('default icons', None) - if icons_setting == 'font awesome': - m = re.search(r'^(fa[a-z])-fa-(.*)', text) - if m: - the_prefix = m.group(1) - text = m.group(2) - else: - the_prefix = docassemble.base.functions.get_config('font awesome prefix', 'fa-solid') - if the_prefix == 'fab': - the_prefix = 'fa-brands' - elif the_prefix == 'far': - the_prefix = 'fa-regular' - elif the_prefix == 'fas': - the_prefix = 'fa-solid' - return '' - if icons_setting == 'material icons': - return '' + str(text) + '' - return None - - -def emoji_html(text, status=None, question=None, images=None): - # logmessage("Got to emoji_html") - if status is not None and question is None: - question = status.question - if images is None: - images = question.interview.images - if text in images: - if status is not None and images[text].attribution is not None: - status.attributions.add(images[text].attribution) - return image_url(images[text].get_reference(), word('icon'), '1em', emoji=True, question=question) - icon_html = get_icon_html(text) - if icon_html: - return icon_html - return ":" + str(text) + ":" - - -def emoji_insert(text, status=None, images=None): - if images is None: - images = status.question.interview.images - if text in images: - if status is not None and images[text].attribution is not None: - status.attributions.add(images[text].attribution) - return "[EMOJI " + images[text].get_reference() + ', 1.2em]' - return ":" + str(text) + ":" - - -def link_rewriter(m, status): - the_path = None - if m.group(1).startswith('#'): - return ' 0: - lang = docassemble.base.functions.get_language() - for term in question.terms: - terms_done.add(term.lower()) - # logmessage("Searching for term " + term + " in " + a) - if lang in question.terms[term]['re']: - a = question.terms[term]['re'][lang].sub(sub_term, a) - else: - a = question.terms[term]['re'][question.language].sub(sub_term, a) - # logmessage("string is now " + str(a)) - if len(question.autoterms) > 0: - lang = docassemble.base.functions.get_language() - for term in question.autoterms: - if term.lower() in terms_done: - continue - terms_done.add(term.lower()) - # logmessage("Searching for term " + term + " in " + a) - if lang in question.autoterms[term]['re']: - a = question.autoterms[term]['re'][lang].sub(r'[[\1]]', a) - else: - a = question.autoterms[term]['re'][question.language].sub(r'[[\1]]', a) - # logmessage("string is now " + str(a)) - if 'interview_terms' in status.extras: - interview_terms = status.extras['interview_terms'] - else: - interview_terms = question.interview.terms - if 'interview_autoterms' in status.extras: - interview_autoterms = status.extras['interview_autoterms'] - else: - interview_autoterms = question.interview.autoterms - else: - interview_terms = question.interview.terms - interview_autoterms = question.interview.autoterms - if len(interview_terms) > 0: - lang = docassemble.base.functions.get_language() - if lang in interview_terms and len(interview_terms[lang]) > 0: - for term in interview_terms[lang]: - if term.lower() in terms_done: - continue - terms_done.add(term.lower()) - # logmessage("Searching for term " + term + " in " + a) - a = interview_terms[lang][term]['re'].sub(sub_term, a) - # logmessage("string is now " + str(a)) - elif question.language in interview_terms and len(interview_terms[question.language]) > 0: - for term in interview_terms[question.language]: - if term.lower() in terms_done: - continue - terms_done.add(term.lower()) - # logmessage("Searching for term " + term + " in " + a) - a = interview_terms[question.language][term]['re'].sub(sub_term, a) - # logmessage("string is now " + str(a)) - if len(interview_autoterms) > 0: - lang = docassemble.base.functions.get_language() - if lang in interview_autoterms and len(interview_autoterms[lang]) > 0: - for term in interview_autoterms[lang]: - if term.lower() in terms_done: - continue - terms_done.add(term.lower()) - # logmessage("Searching for term " + term + " in " + a) - a = interview_autoterms[lang][term]['re'].sub(r'[[\1]]', a) - # logmessage("string is now " + str(a)) - elif question.language in interview_autoterms and len(interview_autoterms[question.language]) > 0: - for term in interview_autoterms[question.language]: - if term.lower() in terms_done: - continue - terms_done.add(term.lower()) - # logmessage("Searching for term " + term + " in " + a) - a = interview_autoterms[question.language][term]['re'].sub(r'[[\1]]', a) - # logmessage("string is now " + str(a)) - a = html_filter(str(a), status=status, question=question, embedder=embedder, default_image_width=default_image_width, external=external) - # logmessage("before: " + a) - if status and status.extras.get('tableCssClass', None): - classes = status.extras['tableCssClass'].split(',') - table_class = json.dumps(classes[0].strip()) - if len(classes) > 1: - thead_class = json.dumps(classes[1].strip()) - else: - thead_class = None - else: - table_class = server.default_table_class - thead_class = server.default_thead_class - a = re.sub(r'<(/?)table', r'<\1TABLE', a) - a = re.sub(r'', r'', a) - if use_pandoc: - converter = pandoc.MyPandoc() - converter.output_format = 'html' - converter.input_content = a - converter.convert(question) - result = converter.output_content - else: - try: - result = docassemble.base.functions.this_thread.markdown.reset().convert(a) - except: - # Try again because sometimes it fails randomly and maybe trying again will work. - result = docassemble.base.functions.this_thread.markdown.reset().convert(a) - result = re.sub(r'', r'
', result) - if thead_class: - result = re.sub(r'', r'', result) - result = re.sub(r'
', r'', result) - result = re.sub(r'<(/?)TABLE', r'<\1table', result) - result = re.sub(r'', r'', result) - result = re.sub(r'<(t[dh]) align="(right|left|center)">', r'<\1 class="text-\2">', result) - result = re.sub(r'
', r'
', result) - result = re.sub(r' 0 and 'terms' in status.extras: - result = term_match.sub((lambda x: add_terms(x.group(1), status.extras['terms'], label=x.group(2), status=status, question=question)), result) - if len(question.autoterms) > 0 and 'autoterms' in status.extras: - result = term_match.sub((lambda x: add_terms(x.group(1), status.extras['autoterms'], label=x.group(2), status=status, question=question)), result) - if 'interview_terms' in status.extras: - interview_terms = status.extras['interview_terms'] - else: - interview_terms = question.interview.terms - if 'interview_autoterms' in status.extras: - interview_autoterms = status.extras['interview_autoterms'] - else: - interview_autoterms = question.interview.autoterms - else: - interview_terms = question.interview.terms - interview_autoterms = question.interview.autoterms - if lang in interview_terms and len(interview_terms[lang]): - result = term_match.sub((lambda x: add_terms(x.group(1), interview_terms[lang], label=x.group(2), status=status, question=question)), result) - elif question.language in interview_terms and len(interview_terms[question.language]): - result = term_match.sub((lambda x: add_terms(x.group(1), interview_terms[question.language], label=x.group(2), status=status, question=question)), result) - if lang in interview_autoterms and len(interview_autoterms[lang]): - result = term_match.sub((lambda x: add_terms(x.group(1), interview_autoterms[lang], label=x.group(2), status=status, question=question)), result) - elif question.language in interview_autoterms and len(interview_autoterms[question.language]): - result = term_match.sub((lambda x: add_terms(x.group(1), interview_autoterms[question.language], label=x.group(2), status=status, question=question)), result) - do_not_scan_for_emojis = bool(re.search(r'\[NO_EMOJIS\]', result)) - if do_not_scan_for_emojis: - result = re.sub(r'\[NO_EMOJIS\]\s*', r'', result) - if status is not None and question.interview.scan_for_emojis and not do_not_scan_for_emojis: - result = emoji_match.sub((lambda x: emoji_html(x.group(1), status=status, question=question)), result) - result = re.sub(r'