transparencybc.html

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>TransparencyBC Case Study — Thomas Roy</title>
    <meta name="description" content="Technical case study of TransparencyBC: architecture and implementation walkthrough of a public-records search tool built with Next.js, Python/Flask search services, Typesense chunk indexing, and PostgreSQL.">
    <link rel="preconnect" href="https://fonts.googleapis.com">
    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
    <link href="https://fonts.googleapis.com/css2?family=Syne:wght@400;500;600;700;800&family=Inter:ital,wght@0,300;0,400;0,500;1,300&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
    <link rel="icon" href="favicon.svg" type="image/svg+xml">
    <link rel="stylesheet" href="css/style.css">
</head>
<body>

    <!-- ── NAV ──────────────────────────────────────────── -->
    <header class="nav" id="nav">
        <div class="nav-inner">
            <a href="index.html" class="nav-logo">TR</a>
            <nav class="nav-links">
                <a href="index.html#work" class="nav-link">&#8592; All Projects</a>
                <a href="resume/Thomas_Roy_Resume.pdf" class="nav-link" target="_blank" rel="noreferrer">Resume</a>
                <a href="https://transparencybc.com/" class="nav-cta" target="_blank" rel="noreferrer">Live Application &#8599;</a>
            </nav>
        </div>
    </header>

    <main class="case-study">

        <!-- ── HERO ─────────────────────────────────────── -->
        <header class="case-study-hero">
            <a href="index.html#work" class="case-study-back">&#8592; Back to Projects</a>
            <p class="section-label reveal">Case Study · Project Architecture</p>
            <h1 class="case-study-title reveal" data-delay="80">TransparencyBC:<br>Architecture &amp; Engineering Walkthrough</h1>
            <p class="case-study-lead reveal" data-delay="140">
                A technical breakdown of a public-records search platform: decoupling
                a Python/Flask search API, chunked Typesense indexing, structured document
                summaries, and signed document streaming for British Columbia FOI disclosures.
            </p>

            <div class="case-study-meta-grid reveal" data-delay="200">
                <div class="meta-item">
                    <span class="meta-item-label">Role</span>
                    <span class="meta-item-value">Full-Stack &amp; Systems Developer</span>
                </div>
                <div class="meta-item">
                    <span class="meta-item-label">Architecture</span>
                    <span class="meta-item-value">Python (Flask) · Typesense · PostgreSQL · Next.js</span>
                </div>
                <div class="meta-item">
                    <span class="meta-item-label">Project Status</span>
                    <span class="meta-item-value">Live application at transparencybc.com (paid subscription required)</span>
                </div>
                <div class="meta-item">
                    <span class="meta-item-label">Walkthrough Scope</span>
                    <span class="meta-item-value">Illustrative static walkthrough with screenshots from the project</span>
                </div>
            </div>
        </header>

        <!-- ── BOUNDARIES CALLOUT ───────────────────────── -->
        <section class="boundaries-banner reveal">
            <h2 class="boundaries-title">System Boundaries &amp; Access Model</h2>
            <div class="boundaries-grid">
                <div class="boundary-card">
                    <span class="boundary-tag">Illustrative Walkthrough</span>
                    <h4>Public Case Study (This Page)</h4>
                    <p>
                        Architecture walkthrough and static screenshots from the project
                        available for technical and recruiter evaluation without requiring an account.
                    </p>
                </div>
                <div class="boundary-card">
                    <span class="boundary-tag">Commercial Service</span>
                    <h4>Paid Live Application</h4>
                    <p>
                        The live service at <a href="https://transparencybc.com/" target="_blank" rel="noreferrer" style="color: var(--accent); text-decoration: underline;">transparencybc.com</a> is a commercial application for researchers and journalists; searching and reading full documents on the live site requires active subscription credentials.
                    </p>
                </div>
                <div class="boundary-card">
                    <span class="boundary-tag">Private Codebase</span>
                    <h4>Nonpublic Source &amp; Pipelines</h4>
                    <p>
                        The document scrapers, OCR processing pipelines, and proprietary indexing scripts reside in private repositories and are not public.
                    </p>
                </div>
            </div>
        </section>

        <!-- ── SECTION 1: THE PROBLEM ───────────────────── -->
        <section class="case-section reveal">
            <div class="case-section-header">
                <p class="section-label">01 / The Problem</p>
                <h2 class="case-section-title">Government disclosures distributed across unindexed scans</h2>
            </div>
            <p class="case-prose">
                British Columbia government Freedom of Information (FOI) releases, Office of the Information and Privacy Commissioner (OIPC) orders, and judicial reviews contain important public records. However, records are published across separate portals as scanned, multi-page PDFs with varying visual quality and no centralized full-text search index.
            </p>
            <p class="case-prose">
                Researchers and journalists frequently need to locate specific topics or cited statutory exemptions—such as <strong>FOIPPA Section 13</strong> (policy advice), <strong>Section 14</strong> (legal advice), <strong>Section 15</strong> (law enforcement), <strong>Section 21</strong> (third-party business interests), or <strong>Section 22</strong> (personal privacy). Without centralized search, discovering relevant files required manually checking disclosure logs, downloading separate bundles, and reviewing pages by hand.
            </p>
            <p class="case-prose">
                TransparencyBC was designed to address this workflow: an ingestion and OCR pipeline, a dedicated search API backed by Typesense and PostgreSQL, structured document summaries, and a web-based PDF viewer.
            </p>
        </section>

        <!-- ── SECTION 2: ARCHITECTURE & DATA FLOW ──────── -->
        <section class="case-section reveal">
            <div class="case-section-header">
                <p class="section-label">02 / Architecture &amp; Data Flow</p>
                <h2 class="case-section-title">Decoupled search microservice and chunked indexing</h2>
            </div>
            <p class="case-prose">
                To isolate search queries from batch scraping and OCR workloads, the system separates background data ingestion from the user-facing search API:
            </p>

            <div class="arch-diagram">
                <div class="arch-flow">
                    <div class="arch-stage">
                        <span class="arch-stage-num">STAGE 01</span>
                        <h4>Ingestion &amp; OCR (Private)</h4>
                        <p>Scrapers poll disclosure logs, retrieve release packages, and run OCR normalization to extract text and record metadata.</p>
                        <div class="arch-pills">
                            <span class="arch-pill">Python Scrapers</span>
                            <span class="arch-pill">OCR Pipeline</span>
                            <span class="arch-pill">Private Runner</span>
                        </div>
                    </div>
                    <div class="arch-stage">
                        <span class="arch-stage-num">STAGE 02</span>
                        <h4>Search Service API</h4>
                        <p>Standalone Flask service (<code>search_service_api</code>) querying Typesense for chunk-level text matching and PostgreSQL for document records and metadata.</p>
                        <div class="arch-pills">
                            <span class="arch-pill">Python / Flask</span>
                            <span class="arch-pill">Typesense</span>
                            <span class="arch-pill">PostgreSQL Corpus</span>
                        </div>
                    </div>
                    <div class="arch-stage">
                        <span class="arch-stage-num">STAGE 03</span>
                        <h4>Next.js Application</h4>
                        <p>Web frontend with API proxy routes, account authentication, structured document summaries, and signed PDF document streaming.</p>
                        <div class="arch-pills">
                            <span class="arch-pill">Next.js</span>
                            <span class="arch-pill">TypeScript</span>
                            <span class="arch-pill">Signed Streaming</span>
                        </div>
                    </div>
                </div>
            </div>

            <p class="case-prose">
                <strong>Data flow lifecycle:</strong>
            </p>
            <ul class="project-list" style="margin-bottom: 24px;">
                <li><strong>Ingestion:</strong> Scrapers collect release packages and record metadata (request number, ministry, date, exemptions cited).</li>
                <li><strong>Chunking &amp; Indexing:</strong> Document text is segmented into searchable chunks in Typesense (bundle collection and response-package scope) so full-text queries match indexed chunks rather than reading multi-megabyte PDFs at request time.</li>
                <li><strong>API Proxying:</strong> Next.js backend routes (<code>/api/search</code> and <code>/api/search-preview</code>) forward requests to <code>search_service_api</code>, passing internal authentication headers and isolating backend credentials.</li>
                <li><strong>Signed Document Streaming:</strong> Document viewing routes verify access and stream PDF files with time-limited HMAC signatures and <code>no-store</code> cache headers instead of exposing static public storage links.</li>
            </ul>
        </section>

        <!-- ── SECTION 3: SCREENSHOTS ───────────────────── -->
        <section class="case-section reveal">
            <div class="case-section-header">
                <p class="section-label">03 / Interface Walkthrough</p>
                <h2 class="case-section-title">Screenshots from the project</h2>
            </div>
            <p class="case-prose">
                Below are static screenshots captured from the project interface, illustrating the search desk, structured document summary, and PDF viewer.
            </p>

            <!-- Figure 1: Search Desk -->
            <figure class="case-figure">
                <div class="case-figure-img-wrap">
                    <img src="images/transparencybc_search.png" alt="Static screenshot of the TransparencyBC Search Desk interface showing topic filters, date pickers, and saved search tags" class="case-figure-img">
                </div>
                <figcaption class="case-figure-caption">
                    <div class="caption-label-row">
                        <span class="caption-title">Figure 1: Search Desk Interface (Static Screenshot)</span>
                        <span class="caption-tag">Static Screenshot · Search Interface</span>
                    </div>
                    <p class="caption-desc">
                        The Search Desk interface provides text search across BC Gov FOI releases, OIPC orders, and judicial reviews. Controls include source filtering, date ranges (From / To), and saved search tags (such as <code>housing</code>, <code>contractor</code>, and <code>site c</code>).
                    </p>
                </figcaption>
            </figure>

            <!-- Figure 2: AI Summary -->
            <figure class="case-figure">
                <div class="case-figure-img-wrap">
                    <img src="images/transparencybc_summary.png" alt="Static screenshot of a document bundle summary panel for FOI request EML-2020-07279" class="case-figure-img">
                </div>
                <figcaption class="case-figure-caption">
                    <div class="caption-label-row">
                        <span class="caption-title">Figure 2: Document Summary Panel (Static Screenshot)</span>
                        <span class="caption-tag">Static Screenshot · Document Metadata</span>
                    </div>
                    <p class="caption-desc">
                        Summary view for FOI request <code>EML-2020-07279</code> (Ministry of Energy, Mines and Low Carbon Innovation regarding Valentine Mountain Mine permits). Displays extracted metadata including core event, major topics, overarching themes, notable facts, and cited FOIPPA statutory exemptions (Sections 13, 14, 15, 21, and 22).
                    </p>
                </figcaption>
            </figure>

            <!-- Figure 3: Viewer -->
            <figure class="case-figure">
                <div class="case-figure-img-wrap">
                    <img src="images/transparencybc_viewer.png" alt="Static screenshot of the document viewer displaying an official FOIPPA response package with page navigation and zoom controls" class="case-figure-img">
                </div>
                <figcaption class="case-figure-caption">
                    <div class="caption-label-row">
                        <span class="caption-title">Figure 3: Document Viewer (Static Screenshot)</span>
                        <span class="caption-tag">Static Screenshot · Document Viewer</span>
                    </div>
                    <p class="caption-desc">
                        Document viewer displaying page 1 of 5 of an official FOIPPA response package. Features tabbed navigation between response letter and package, page navigation, zoom controls, and in-document text search across OCR text.
                    </p>
                </figcaption>
            </figure>
        </section>

        <!-- ── SECTION 4: KEY ENGINEERING DECISIONS ─────── -->
        <section class="case-section reveal">
            <div class="case-section-header">
                <p class="section-label">04 / Engineering Decisions</p>
                <h2 class="case-section-title">Architectural trade-offs &amp; implementation choices</h2>
            </div>

            <div class="decisions-grid">
                <div class="decision-card">
                    <h4>1. Decoupled Search API vs. Ingestion Workload</h4>
                    <p>
                        <strong>Decision:</strong> Separated <code>search_service_api</code> into an independent Flask service, distinct from scraper and OCR tasks.
                    </p>
                    <p>
                        <strong>Tradeoff:</strong> Scraper tasks and OCR processing incur heavy CPU and disk I/O. Decoupling search API execution isolates resource contention so background ingestion does not directly tie up web query processes.
                    </p>
                </div>

                <div class="decision-card">
                    <h4>2. Chunk-Based Full-Text Indexing</h4>
                    <p>
                        <strong>Decision:</strong> Rather than indexing entire 200-page release packages as single monolithic documents, text is split into granular chunks stored in Typesense.
                    </p>
                    <p>
                        <strong>Tradeoff:</strong> Chunked indexing enables snippet highlighting and avoids reading or parsing full PDF files during search queries, keeping index memory usage predictable.
                    </p>
                </div>

                <div class="decision-card">
                    <h4>3. Signed PDF Streaming Routes</h4>
                    <p>
                        <strong>Decision:</strong> PDFs are streamed through backend routes using HMAC-signed tokens rather than exposing direct public storage links.
                    </p>
                    <p>
                        <strong>Tradeoff:</strong> Public storage links bypass application-level access controls and allow unauthenticated scraping. Signed streaming routes verify session permissions and prevent unauthenticated direct file access.
                    </p>
                </div>

                <div class="decision-card">
                    <h4>4. PostgreSQL State Separation</h4>
                    <p>
                        <strong>Decision:</strong> Maintained separate PostgreSQL databases for corpus metadata and user application state.
                    </p>
                    <p>
                        <strong>Tradeoff:</strong> Avoids SQLite concurrency locking under web queries and isolates schema changes between document corpus records and user account data.
                    </p>
                </div>
            </div>
        </section>

        <!-- ── FOOTER ACTIONS ───────────────────────────── -->
        <div class="case-study-footer-actions reveal">
            <a href="index.html#work" class="btn btn-primary">&#8592; Back to All Projects</a>
            <div class="project-links">
                <a href="resume/Thomas_Roy_Resume.pdf" class="project-link" target="_blank" rel="noreferrer">Open Resume &#8594;</a>
                <a href="https://github.com/ThomasRoyProjects" class="project-link" target="_blank" rel="noreferrer">GitHub Profile &#8594;</a>
                <a href="https://transparencybc.com/" class="project-link" target="_blank" rel="noreferrer">Live Application &#8599;</a>
            </div>
        </div>

    </main>

    <!-- ── FOOTER ───────────────────────────────────────── -->
    <footer class="site-footer">
        <div class="footer-inner">
            <span class="footer-brand">Thomas Roy</span>
            <span class="footer-copy">&#169; 2026</span>
            <a href="resume/Thomas_Roy_Resume.pdf" class="footer-link" target="_blank" rel="noreferrer">Resume</a>
            <a href="https://github.com/ThomasRoyProjects" class="footer-link" target="_blank" rel="noreferrer">GitHub</a>
        </div>
    </footer>

    <script src="hero.js"></script>
</body>
</html>