T3646 FIX: Compose letter issues, allow overlay on project gift letters

sbc_compassion/models/correspondence.py

@@ -15,6 +15,7 @@
 import subprocess
 import uuid
 from collections import defaultdict
+from io import BytesIO
 
 from dateutil.relativedelta import relativedelta
 from psycopg2 import OperationalError
@@ -155,7 +156,22 @@ class Correspondence(models.Model):
 
     # 2. Attachments and scans
     ##########################
-    sponsor_letter_scan = fields.Binary()
+    sponsor_letter_scan = fields.Binary(
+        string="Final letter PDF",
+        help="Final PDF of the letter. It is sent as it is, nothing is "
+        "overlaid on it, and the Source PDF is ignored.",
+    )
+    overlay_source_pdf = fields.Binary(
+        string="Source PDF",
+        help="Original PDF of a letter that was loaded manually. It is never "
+        "modified: the translation is composed on top of it when the letter "
+        "is printed, so it can be edited and composed again. It is only used "
+        "while there is no final letter PDF, which is used as it is.",
+    )
+    overlay_source_file_name = fields.Char(compute="_compute_overlay_source_file_name")
+    overlay_source_nbr_pages = fields.Integer(
+        compute="_compute_overlay_source_nbr_pages", store=True
+    )
     file_name = fields.Char(compute="_compute_file_name", store=True)
     letter_format = fields.Selection(
         [("pdf", "pdf"), ("zip", "zip")],
@@ -443,6 +459,30 @@ def _get_text(self, source_text):
         )
         return ("\n" + PAGE_SEPARATOR + "\n").join(txt)
 
+    @api.depends("file_name")
+    def _compute_overlay_source_file_name(self):
+        for letter in self:
+            name = (letter.file_name or letter.name or "letter").rsplit(".", 1)[0]
+            letter.overlay_source_file_name = f"{name} - Source.pdf"
+
+    @api.depends("overlay_source_pdf")
+    def _compute_overlay_source_nbr_pages(self):
+        for letter in self:
+            nbr_pages = 0
+            if letter.overlay_source_pdf:
+                try:
+                    pdf_data = base64.b64decode(
+                        letter.with_context(bin_size=False).overlay_source_pdf
+                    )
+                    nbr_pages = PdfFileReader(BytesIO(pdf_data)).numPages
+                except Exception:
+                    _logger.warning(
+                        "Could not read the source PDF of letter %s",
+                        letter.id,
+                        exc_info=True,
+                    )
+            letter.overlay_source_nbr_pages = nbr_pages
+
     @api.depends("sponsor_letter_scan")
     def _compute_letter_format(self):
         for letter in self:
@@ -695,6 +735,7 @@ def create(self, vals_list):
                 )
 
         letters = super().create(vals_list)
+        letters._create_overlay_source_pages()
         # T1676 : Each page should contain at least one textbox (paragraph)
         letters.create_text_boxes()
         # The report overlays the original text on the template pages, so the
@@ -750,6 +791,8 @@ def write(self, vals):
             )
 
         super().write(vals)
+        if vals.get("overlay_source_pdf"):
+            self._create_overlay_source_pages()
         if "translation_language_id" in vals or "page_ids" in vals:
             if not self.mapped("page_ids.paragraph_ids"):
                 self.create_text_boxes()
@@ -948,8 +991,14 @@ def get_pdf(self):
 
     def _prints_translation_on_pages(self):
         """The stored PDF of GMC doesn't hold the translation we print on the
-        page images. Without all the images, keep the stored PDF."""
+        page images. Without all the images, keep the stored PDF.
+        A source PDF is composed with the translation, unless a final PDF is
+        stored."""
         self.ensure_one()
+        if self.overlay_source_pdf:
+            # The source PDF is meant to get the translation printed on it,
+            # until a final PDF is set: it is then kept as it is.
+            return not self.sponsor_letter_scan
         pages = self.page_ids.filtered("final_page_url")
         return bool(
             self.report_needs_final_text
@@ -1533,6 +1582,12 @@ def _fix_missing_pages(self):
                     letter.cloudinary_final_letter_url = cloudinary_final
                     letter.sponsor_letter_scan = False
 
+    def _create_overlay_source_pages(self):
+        """Each page of the source PDF needs a letter page to compose on."""
+        for letter in self.filtered("overlay_source_pdf"):
+            for _i in range(len(letter.page_ids), letter.overlay_source_nbr_pages):
+                letter.page_ids.create({"correspondence_id": letter.id})
+
     def _create_missing_pages(self, number_pages=0):
         self.ensure_one()
         if self.page_ids:

sbc_compassion/models/correspondence_page.py

@@ -19,6 +19,7 @@
 from odoo import api, fields, models
 
 from ..tools.onramp_connector import SBCConnector
+from .correspondence_text_box import PAGE_HEIGHT_PX, PAGE_WIDTH_PX
 
 _logger = logging.getLogger(__name__)
 
@@ -78,25 +79,29 @@ def _compute_page_image(self, image_type="original"):
         image_field = f"{image_type}_page_image"
         url_field = f"{image_type}_page_url"
         for page in self:
-            if getattr(page, cloudinary_url_field):
-                setattr(
-                    page,
-                    image_field,
-                    page._get_cloundinary_image(getattr(page, cloudinary_url_field)),
-                )
-            if not getattr(page, image_field) and getattr(page, url_field):
-                connector = SBCConnector(self.env)
-                setattr(
-                    page,
-                    image_field,
-                    connector.get_letter_image(getattr(page, url_field), {"dpi": 96}),
-                )
-            if (
-                not getattr(page, image_field)
-                and page.correspondence_id.direction == "Beneficiary To Supporter"
-            ):
+            cloudinary_url = getattr(page, cloudinary_url_field)
+            page_url = getattr(page, url_field)
+            image = False
+            # A cropped image slices a stack of pages in equal parts, which is
+            # wrong when the pages don't have the same height: the page of the
+            # document is used first.
+            connect_first = bool(cloudinary_url and "c_crop" in cloudinary_url)
+            if connect_first:
+                image = page._get_connect_image(page_url)
+            if not image and cloudinary_url:
+                image = page._get_cloundinary_image(cloudinary_url)
+            if not image and not connect_first:
+                image = page._get_connect_image(page_url)
+            if image:
+                setattr(page, image_field, image)
+            elif page.correspondence_id.direction == "Beneficiary To Supporter":
                 setattr(page, image_field, page.template_id.background)
 
+    def _get_connect_image(self, page_url):
+        if not page_url:
+            return False
+        return SBCConnector(self.env).get_letter_image(page_url, {"dpi": 96}) or False
+
     def _compute_original_page_image(self):
         self._compute_page_image("original")
 
@@ -126,8 +131,13 @@ def _compute_template_page(self):
                 modulo_page = template.page_ids[new_index : new_index + 1]
                 if page.correspondence_id.direction == "Supporter To Beneficiary":
                     page_template = modulo_page
-                elif page.original_page_url:
-                    # Extra scanned page of the child: no text box over it
+                elif (
+                    page.original_page_url
+                    or page.page_index
+                    <= page.correspondence_id.overlay_source_nbr_pages
+                ):
+                    # Extra scanned page of the child or page of the source
+                    # PDF: no text box over it
                     page_template = template.page_ids.browse()
                 else:
                     page_template = template.additional_page_id or modulo_page
@@ -142,6 +152,10 @@ def fetch_image(self):
         """
         self.ensure_one()
         letter = self.correspondence_id
+        if self.page_index <= letter.overlay_source_nbr_pages:
+            return self._fetch_pdf_page_image(
+                letter.with_context(bin_size=False).overlay_source_pdf, 150
+            )
         if letter.direction == "Supporter To Beneficiary" and not letter.kit_identifier:
             if self.template_id.background:
                 return self.template_id.background
@@ -157,12 +171,41 @@ def fetch_image(self):
             self._compute_page_image(image_field.replace("_page_image", ""))
         return getattr(self, image_field)
 
+    @api.model
+    def fetch_image_width(self, image):
+        """
+        Used by the PDF report: width in px at which the image is printed so
+        that it fits on one A4 page. The width must be absolute, because a
+        percentage resolves to the natural size of the image in wkhtmltopdf,
+        which makes large images overflow on the next page. Images that are
+        higher than A4 are shrunk, the others keep the full page width.
+        """
+        if not image:
+            return PAGE_WIDTH_PX
+        try:
+            width, height = Image.open(BytesIO(base64.b64decode(image))).size
+        except Exception:
+            _logger.warning("Could not read the size of a page image", exc_info=True)
+            return PAGE_WIDTH_PX
+        if not width or not height:
+            return PAGE_WIDTH_PX
+        # Keep a small margin so that rounding never pushes it onto a new page
+        max_width = (PAGE_HEIGHT_PX - 2) * width / height
+        return round(min(PAGE_WIDTH_PX, max_width), 1)
+
     def _fetch_sponsor_pdf_image(self):
         self.ensure_one()
-        if self.correspondence_id.sponsor_letter_scan:
-            pdf_data = base64.b64decode(self.correspondence_id.sponsor_letter_scan)
+        return self._fetch_pdf_page_image(
+            self.correspondence_id.sponsor_letter_scan, 96
+        )
+
+    def _fetch_pdf_page_image(self, pdf, dpi):
+        """Image of the page of a PDF corresponding to this letter page."""
+        self.ensure_one()
+        if pdf:
+            pdf_data = base64.b64decode(pdf)
             image = convert_from_bytes(
-                pdf_data, 96, first_page=self.page_index, last_page=self.page_index
+                pdf_data, dpi, first_page=self.page_index, last_page=self.page_index
             )[0]
             return self.env["import.letters.history"].create_preview(image)
         return False

sbc_compassion/models/correspondence_paragraph.py

@@ -16,6 +16,8 @@
 
 BOX_SEPARATOR = "#BOX#"
 PAGE_SEPARATOR = "#PAGE#"
+# Shorter texts are leftovers (dots, initials), as in v14
+MIN_TEXT_LENGTH = 5
 
 
 class CorrespondenceParagraph(models.Model):
@@ -72,6 +74,20 @@ def fetch_style(self, field_name):
         self.ensure_one()
         return self.get_text_box(field_name).css_style
 
+    def report_lines(self, field_name):
+        """
+        Lines of the text printed in the report. Several blank lines in a row
+        are reduced to one, so that two paragraphs are at most 2 returns apart.
+        """
+        self.ensure_one()
+        lines = []
+        for line in (self[field_name] or "").split("\n"):
+            if line.strip() or (lines and lines[-1].strip()):
+                lines.append(line)
+        while lines and not lines[-1].strip():
+            lines.pop()
+        return lines
+
     def check_overflow(self, field_name):
         """
         Check if the text overflows the text box.
@@ -86,6 +102,8 @@ def check_overflow(self, field_name):
         if not text_box:
             # The page has no box for this text (e.g. B2S L4/L6 print the
             # translation on page 2 only): move it to the next box.
+            if len(text.strip()) < MIN_TEXT_LENGTH:
+                return text, ""
             return "", text
         try:
             return text_box.split_text(text)

sbc_compassion/models/correspondence_text_box.py

@@ -25,6 +25,8 @@
 PAGE_HEIGHT_PX = 1122.5
 FONT_SIZE_PX = 16
 PARAGRAPH_MARGIN_PX = 18.9
+# A blank line adds one line to the gap between two paragraphs (max 2 returns)
+BLANK_LINE_HEIGHT_PX = 19.2
 FONT_FILE = "web/static/fonts/lato/Lato-Reg-webfont.ttf"
 # Emojis and symbols are missing from Lato and rendered wider with a fallback font.
 FALLBACK_GLYPH_WIDTH_EM = 1.25
@@ -136,9 +138,12 @@ def split_text(self, text):
                 nb_lines = max(0, int((max_height - height) // line_height))
                 height += len(lines) * line_height
             else:
-                # An empty paragraph keeps the height of one character.
+                # A blank line has no margin: it adds one line to the gap.
                 nb_lines = 0
-                height += FONT_SIZE_PX
+                height += BLANK_LINE_HEIGHT_PX
+                if height > max_height:
+                    return self._cut_text(paragraphs, index, lines, nb_lines, max_width)
+                continue
             if height > max_height:
                 return self._cut_text(paragraphs, index, lines, nb_lines, max_width)
             height += PARAGRAPH_MARGIN_PX

sbc_compassion/models/ir_actions_report.py

@@ -23,6 +23,12 @@ def _render_qweb_pdf(self, report_ref, res_ids=None, data=None):
         ):
             return super()._render_qweb_pdf(report_ref, res_ids=res_ids, data=data)
 
+        # As the v14 composition did, lay the translation out in the boxes
+        # before printing, whatever page it was saved on. Writes only once.
+        letters = self.env["correspondence"].sudo().browse(res_ids)
+        for letter in letters.filtered("report_needs_final_text"):
+            letter.spread_text_to_pages(["translated_text"])
+
         attachments = self.env["ir.attachment"].search(
             [
                 ("res_model", "=", "correspondence"),

sbc_compassion/reports/correspondence_report.xml

@@ -26,24 +26,30 @@
 }
 .overlay p {
   margin: 0 0 5mm;
-  min-height: 1em;
+}
+/* A blank line adds one line to the gap, see BLANK_LINE_HEIGHT_PX */
+.overlay p.blank {
+  margin: 0;
+  min-height: 19.2px;
 }
       </style>
             <t t-foreach="docs" t-as="letter">
                 <t t-foreach="letter.page_ids" t-as="page">
                     <div class="page" style="page-break-after:always;">
+                        <t
+              t-set="background"
+              t-value="page.fetch_image()"
+            />
+                        <!-- Pages higher than A4 are shrunk to fit on the page,
+                             the text boxes follow as they are in % of the image -->
                         <div
-              class="image-container w-100"
-              style="position: relative;display: inline-block;"
+              class="image-container"
+              t-attf-style="position: relative; display: block; margin: 0 auto; width: {{page.fetch_image_width(background)}}px;"
             >
-                            <t
-                t-set="background"
-                t-value="page.fetch_image()"
-              />
                             <img
                 t-if="background"
                 t-attf-src="data:image/jpeg;base64,{{background}}"
-                class="w-100"
+                style="display: block; width: 100%;"
               />
                             <t t-if="letter.report_needs_overlay">
                                 <div
@@ -65,10 +71,10 @@
                     t-att-style="text_box.fetch_style('original_text')"
                   >
                                         <t
-                      t-foreach="(text_box.original_text or '').split('\n')"
+                      t-foreach="text_box.report_lines('original_text')"
                       t-as="p"
                     >
-                                            <p t-out="p" />
+                                            <p t-att-class="'blank' if not p.strip() else None" t-out="p" />
                                         </t>
                                     </div>
                                     <div
@@ -77,10 +83,10 @@
                     t-att-style="text_box.fetch_style('translated_text')"
                   >
                                         <t
-                      t-foreach="(text_box.translated_text or '').split('\n')"
+                      t-foreach="text_box.report_lines('translated_text')"
                       t-as="p"
                     >
-                                            <p t-out="p" />
+                                            <p t-att-class="'blank' if not p.strip() else None" t-out="p" />
                                         </t>
                                     </div>
                                     <div
@@ -89,10 +95,10 @@
                     t-att-style="text_box.fetch_style('english_text')"
                   >
                                         <t
-                      t-foreach="(text_box.english_text or '').split('\n')"
+                      t-foreach="text_box.report_lines('english_text')"
                       t-as="p"
                     >
-                                            <p t-out="p" />
+                                            <p t-att-class="'blank' if not p.strip() else None" t-out="p" />
                                         </t>
                                     </div>
                                 </t>