diff --git a/back-end/alembic/versions/a9e5d21c7b48_add_used_email_token_table.py b/back-end/alembic/versions/a9e5d21c7b48_add_used_email_token_table.py new file mode 100644 index 0000000..435b96e --- /dev/null +++ b/back-end/alembic/versions/a9e5d21c7b48_add_used_email_token_table.py @@ -0,0 +1,36 @@ +"""add used_email_token table + +Revision ID: a9e5d21c7b48 +Revises: f2a91c7e4d08 +Create Date: 2026-07-06 + +One-time-use ledger for email link tokens (verify / reset / join-org): the +token's jti is recorded on first successful use so replays are refused. +""" + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a9e5d21c7b48" +down_revision = "f2a91c7e4d08" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "used_email_token", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("jti", sa.String(length=36), nullable=False), + sa.Column("expires_at", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("jti"), + ) + op.create_index("ix_used_email_token_expires_at", "used_email_token", ["expires_at"]) + + +def downgrade() -> None: + op.drop_index("ix_used_email_token_expires_at", table_name="used_email_token") + op.drop_table("used_email_token") diff --git a/back-end/controllers/database_controller/vt_ops.py b/back-end/controllers/database_controller/vt_ops.py index d96e6c7..04eed5e 100755 --- a/back-end/controllers/database_controller/vt_ops.py +++ b/back-end/controllers/database_controller/vt_ops.py @@ -16,6 +16,7 @@ from controllers.database_controller.kml_ops import get_kml_data from database.models import mbtiles, vector_tiles from database.sessions import Session +from utils.logger_config import logger from utils.settings import DATABASE_URL from .file_ops import ( @@ -203,11 +204,18 @@ def add_values_to_VT(geojson_file_path, mbtiles_file_paths, folderid): def run_tippecanoe(command): - result = subprocess.run(command, shell=True, check=True, stderr=subprocess.PIPE) - - if result.stderr: - print("Tippecanoe stderr:", result.stderr.decode()) - + """Run a tippecanoe/tile-join command, surfacing stderr on BOTH paths. + On failure the raised message must carry the real error: it is what gets + stored as the task result, and a bare "returned non-zero exit status 1" + leaves tile failures undiagnosable without shell access to the worker.""" + result = subprocess.run(command, shell=True, stderr=subprocess.PIPE) + stderr = (result.stderr or b"").decode(errors="replace").strip() + if result.returncode != 0: + logger.error(f"tippecanoe failed (exit {result.returncode}): {stderr}\ncmd: {command}") + raise RuntimeError(f"tippecanoe exit {result.returncode}: {stderr[-2000:]}") + if stderr: + # tippecanoe reports progress/feature counts on stderr even on success. + logger.info(f"tippecanoe stderr: {stderr}") return result.returncode diff --git a/back-end/database/models.py b/back-end/database/models.py index 912e612..adee372 100755 --- a/back-end/database/models.py +++ b/back-end/database/models.py @@ -584,6 +584,20 @@ class audit_log(Base): ip = Column(String, nullable=True) +class used_email_token(Base): + """One-time-use ledger for email link tokens (verify / reset / join-org): + a token's jti is recorded on its first successful use and any replay is + refused. Rows are dead weight once expires_at passes (the JWT itself has + expired by then) and are pruned opportunistically on insert.""" + + __tablename__ = "used_email_token" + __table_args__ = (Index("ix_used_email_token_expires_at", "expires_at"),) + + id = Column(Integer, primary_key=True, autoincrement=True) + jti = Column(String(36), unique=True, nullable=False) + expires_at = Column(DateTime, nullable=False) + + class site_setting(Base): """Site-wide key/value settings the platform operator controls from the admin panel (e.g. site_theme). Not per-org, not user-visible to edit.""" diff --git a/back-end/routes/_email.py b/back-end/routes/_email.py index 377e891..75315e0 100644 --- a/back-end/routes/_email.py +++ b/back-end/routes/_email.py @@ -1,7 +1,9 @@ +import uuid from datetime import UTC, datetime, timedelta import jwt from flask_mail import Message +from sqlalchemy.exc import IntegrityError from utils.flask_app import app, mail @@ -12,15 +14,25 @@ EMAIL_TOKEN_AUDIENCE = "bdk-email" +# How long a verify/reset/join link stays valid. An hour, not minutes: users +# routinely open these emails well after they arrive, and an expired link +# reads as a broken product. Tokens are not single-use, so this is also the +# replay window for an already-used link — keep it to hours, not days. +EMAIL_TOKEN_LIFETIME = timedelta(minutes=60) + + def create_email_token(userid, email, operation, org_id=-1): # Expiry must be timezone-aware UTC: PyJWT treats a naive datetime as UTC, # so naive local time on a non-UTC host mints already-expired tokens. - expiration = datetime.now(UTC) + timedelta(minutes=15) + expiration = datetime.now(UTC) + EMAIL_TOKEN_LIFETIME email_token = jwt.encode( { "sub": {"id": userid, "email": email, "operation": operation, "org_id": org_id}, "exp": expiration, "aud": EMAIL_TOKEN_AUDIENCE, + # One-time-use handle: consume_email_token records it on first + # successful use, so a leaked/reused link can't be replayed. + "jti": uuid.uuid4().hex, }, app.config["JWT_SECRET_KEY"], algorithm="HS256", @@ -28,6 +40,39 @@ def create_email_token(userid, email, operation, org_id=-1): return email_token +def consume_email_token(decoded): + """One-time-use gate. True the first time a decoded token is consumed; + False on replay or when the token has no jti (minted before stamping — + those age out within EMAIL_TOKEN_LIFETIME of the deploy). + + Call it after the domain checks pass and before mutating anything. The + unique constraint on jti is the actual gate, so two concurrent uses of + the same token can't both win. Own short session on purpose: the + IntegrityError rollback must not discard request-scoped work.""" + from database.models import used_email_token + from database.sessions import Session + + jti = decoded.get("jti") + if not jti: + return False + # Column is a naive-UTC DateTime like the rest of the schema. + now = datetime.now(UTC).replace(tzinfo=None) + expires_at = datetime.fromtimestamp(decoded["exp"], UTC).replace(tzinfo=None) + session = Session() + try: + # Opportunistic prune: expired rows guard nothing (their JWTs are + # already dead), so the table stays bounded by the token lifetime. + session.query(used_email_token).filter(used_email_token.expires_at < now).delete() + session.add(used_email_token(jti=jti, expires_at=expires_at)) + session.commit() + return True + except IntegrityError: + session.rollback() + return False + finally: + session.close() + + def send_verification_email_with_token( email, token, title, content, join_org=False, joining_email="", link_path=None ): diff --git a/back-end/routes/admin_ui.py b/back-end/routes/admin_ui.py index ec864da..460ae33 100644 --- a/back-end/routes/admin_ui.py +++ b/back-end/routes/admin_ui.py @@ -7,7 +7,7 @@ """ from flask import Blueprint, g, redirect, render_template, request -from werkzeug.security import check_password_hash, generate_password_hash +from werkzeug.security import generate_password_hash from controllers.database_controller import user_ops from database.sessions import get_session @@ -15,6 +15,7 @@ from services.audit import log_action from utils.admin_auth import ADMIN_COOKIE, mint_admin_session, require_platform_admin from utils.flask_app import limiter +from utils.passwords import needs_rehash, verify_password from utils.settings import IN_PRODUCTION bp = Blueprint("admin_ui", __name__) @@ -66,8 +67,12 @@ def login(): and getattr(user, "is_platform_admin", False) and not getattr(user, "disabled", False) and user.password - and check_password_hash(user.password, password) + and verify_password(user.password, password) ): + if needs_rehash(user.password): + # Legacy (pre-pbkdf2) hash: the user just proved the password, so + # upgrade the stored hash in place — no forced reset. + user_ops.reset_user_password(user.id, password) token, _csrf = mint_admin_session(user.id) response = redirect("/admin/") _set_admin_cookie(response, token) @@ -76,7 +81,7 @@ def login(): # Equalize work on the failure path (absent / non-admin email) so response # time doesn't reveal which emails are platform admins. - check_password_hash(_DUMMY_PASSWORD_HASH, password) + verify_password(_DUMMY_PASSWORD_HASH, password) log_action("admin_login_failed", details={"email": email}) return render_template( "admin/login.html", error="Invalid credentials or not a platform admin." diff --git a/back-end/routes/auth.py b/back-end/routes/auth.py index be58fab..c0b7427 100644 --- a/back-end/routes/auth.py +++ b/back-end/routes/auth.py @@ -10,7 +10,6 @@ verify_jwt_in_request, ) from jwt import ExpiredSignatureError -from werkzeug.security import check_password_hash from controllers.database_controller import ( user_ops, @@ -18,12 +17,14 @@ from database.sessions import get_session from routes._email import ( EMAIL_TOKEN_AUDIENCE, + consume_email_token, create_email_token, send_verification_email_with_token, ) from services.audit import log_action from utils.flask_app import app, limiter from utils.logger_config import logger +from utils.passwords import needs_rehash, verify_password from utils.validation import is_valid_email bp = Blueprint("auth", __name__) @@ -69,6 +70,8 @@ def reset_password(): user_id = decoded_token["sub"]["id"] email = decoded_token["sub"]["email"] if user_ops.verify_user_email(user_id, email, session, False): + if not consume_email_token(decoded_token): + return jsonify({"status": "error", "message": "Invalid token."}), 400 user_ops.reset_user_password(user_id, new_password) return jsonify({"status": "success", "message": "Password reset successfully."}), 200 else: @@ -106,6 +109,9 @@ def verify_token(): logger.debug(operation) + if not consume_email_token(decoded_token): + return jsonify({"status": "error", "message": "Invalid token."}), 400 + setVerified = True if operation == "email_address_verification" else False if user_ops.verify_user_email(user_id, email, session, setVerified): if operation == "join_organization": @@ -189,7 +195,11 @@ def login(): # Needs a try catch to return the correct message user = user_ops.get_user_with_email(email) - if user is not None and check_password_hash(user.password, pword): + if user is not None and verify_password(user.password, pword): + if needs_rehash(user.password): + # Legacy (pre-pbkdf2) hash: the user just proved the password, so + # upgrade the stored hash in place — no forced reset. + user_ops.reset_user_password(user.id, pword) # A soft-disabled account cannot obtain a fresh session. Checked # only after the password verifies, so we never reveal disabled status # to someone who doesn't already hold the credentials. diff --git a/back-end/routes/auth_pages.py b/back-end/routes/auth_pages.py index 6ccd924..9251759 100644 --- a/back-end/routes/auth_pages.py +++ b/back-end/routes/auth_pages.py @@ -15,17 +15,18 @@ set_access_cookies, unset_jwt_cookies, ) -from werkzeug.security import check_password_hash from controllers.database_controller import setting_ops, user_ops from database.sessions import get_session from routes._email import ( EMAIL_TOKEN_AUDIENCE, + consume_email_token, create_email_token, send_verification_email_with_token, ) from services.audit import log_action from utils.flask_app import app, limiter +from utils.passwords import needs_rehash, verify_password from utils.validation import is_valid_email bp = Blueprint("auth_pages", __name__, template_folder="../templates") @@ -53,10 +54,14 @@ def login_submit(): email = (request.form.get("email") or "").strip() pword = request.form.get("password") or "" user = user_ops.get_user_with_email(email) - if user is None or not check_password_hash(user.password, pword): + if user is None or not verify_password(user.password, pword): # Don't leak whether the email exists; record the attempted email. log_action("login_failed", details={"email": email}) return _page("app/auth_login.html", error="Invalid credentials") + if needs_rehash(user.password): + # Legacy (pre-pbkdf2) hash: the user just proved the password, so + # upgrade the stored hash in place — no forced reset. + user_ops.reset_user_password(user.id, pword) if getattr(user, "disabled", False): log_action("login_disabled", user_id=user.id, resource_type="user", resource_id=user.id) return _page("app/auth_login.html", error="This account has been disabled.") @@ -129,6 +134,13 @@ def verify_email(token): ) if decoded["sub"]["operation"] != "email_address_verification": raise ValueError + if not consume_email_token(decoded): + # Re-clicks and mail-scanner prefetches replay this GET link; once + # the account is verified that's a success, not a broken link. + userVal = user_ops.get_user_with_id(decoded["sub"]["id"], session=session) + if userVal is not None and userVal.verified: + return _page("app/auth_verified.html", ok=True, error=None) + raise ValueError if not user_ops.verify_user_email( decoded["sub"]["id"], decoded["sub"]["email"], session, True ): @@ -208,6 +220,12 @@ def reset_token_submit(token): email = decoded["sub"]["email"] if not user_ops.verify_user_email(user_id, email, session, False): raise ValueError + if not consume_email_token(decoded): + return _page( + "app/auth_reset_token.html", + token=token, + error="That link has already been used — request a new one.", + ) user_ops.reset_user_password(user_id, new_password) except jwt.ExpiredSignatureError: return _page( diff --git a/back-end/services/filing_service.py b/back-end/services/filing_service.py index 25b844d..ca54286 100644 --- a/back-end/services/filing_service.py +++ b/back-end/services/filing_service.py @@ -139,6 +139,7 @@ def submission_debts(user_id, folderid, session): submission, in plain words with a place to fix it. Empty list = ready. Feeds the header badge ("N tasks to finish" / "Ready to submit") and the generate sheet — the app's only hard gate.""" + from database.models import celerytaskinfo, fabric_data from database.models import file as file_model from services import plan_service @@ -161,6 +162,49 @@ def submission_debts(user_id, folderid, session): "fix": "/files?tab=fabric", } ) + else: + # The file row is committed synchronously at upload, before — and + # regardless of whether — the worker import succeeds. The gate must + # key on locations actually having landed, or a failed import could + # be submitted as a filing with no fabric-derived served locations. + fabric_imported = ( + session.query(fabric_data.id) + .join(file_model, fabric_data.file_id == file_model.id) + .filter(file_model.folder_id == folderVal.id, file_model.type == "fabric") + .first() + is not None + ) + if not fabric_imported: + # Task status only shapes the wording; the fabric_data rows above + # are the source of truth for readiness. + latest_task = ( + session.query(celerytaskinfo) + .filter( + celerytaskinfo.organization_id == folderVal.organization_id, + celerytaskinfo.operation_type == "Fabric", + celerytaskinfo.folder_deadline == folderVal.deadline, + ) + .order_by(celerytaskinfo.id.desc()) + .first() + ) + if latest_task is not None and latest_task.status in ("PENDING", "STARTED", "RETRY"): + debts.append( + { + "id": "fabric_import", + "label": "Fabric import in progress", + "sub": "locations are still loading — this clears when the import lands", + "fix": "/files?tab=fabric", + } + ) + else: + debts.append( + { + "id": "fabric_import", + "label": "Fabric import failed", + "sub": "the upload didn't import any locations — retry or re-upload it", + "fix": "/files?tab=fabric", + } + ) coverage_files = ( session.query(file_model) diff --git a/back-end/tests/test_email_tokens.py b/back-end/tests/test_email_tokens.py index 6569e59..181db63 100644 --- a/back-end/tests/test_email_tokens.py +++ b/back-end/tests/test_email_tokens.py @@ -77,3 +77,136 @@ def test_verify_email_token(client): assert s.query(user).filter(user.email == "verifyme@example.com").one().verified is True finally: s.close() + + +def test_email_token_lifetime_is_an_hour(client): + """Users often come back to a verify/reset email well after it arrives; + 15 minutes read as "broken link". Pin the intended 60-minute lifetime.""" + import time + + import jwt as pyjwt + + from routes._email import EMAIL_TOKEN_AUDIENCE, create_email_token + from utils.flask_app import app + + token = create_email_token(userid=1, email="ttl@example.com", operation="reset_password") + decoded = pyjwt.decode( + token, + app.config["JWT_SECRET_KEY"], + algorithms=["HS256"], + audience=EMAIL_TOKEN_AUDIENCE, + options={"verify_sub": False}, + ) + lifetime = decoded["exp"] - time.time() + assert 59 * 60 < lifetime <= 60 * 60 + + +def _user_id(email): + from database.models import user + from database.sessions import Session + + s = Session() + try: + return s.query(user).filter(user.email == email).one().id + finally: + s.close() + + +def test_reset_token_is_single_use(client): + from routes._email import create_email_token + + _register(client, "once@example.com") + uid = _user_id("once@example.com") + token = create_email_token(userid=uid, email="once@example.com", operation="reset_password") + + first = client.post("/api/reset_password", json={"token": token, "newPassword": "FirstNew1!"}) + assert first.status_code == 200 + + replay = client.post( + "/api/reset_password", json={"token": token, "newPassword": "AttackerPick1!"} + ) + assert replay.status_code == 400 + + # The replay changed nothing: the first new password still logs in. + login = client.post("/api/login", json={"email": "once@example.com", "password": "FirstNew1!"}) + assert login.get_json()["status"] == "success" + + +def test_verify_token_is_single_use(client): + from routes._email import create_email_token + + _register(client, "onceverify@example.com") + uid = _user_id("onceverify@example.com") + token = create_email_token( + userid=uid, email="onceverify@example.com", operation="email_address_verification" + ) + + assert client.post("/api/verify_token", json={"token": token}).status_code == 200 + assert client.post("/api/verify_token", json={"token": token}).status_code == 400 + + +def test_reset_page_token_is_single_use(client): + from routes._email import create_email_token + + _register(client, "oncepage@example.com") + uid = _user_id("oncepage@example.com") + token = create_email_token(userid=uid, email="oncepage@example.com", operation="reset_password") + + first = client.post(f"/auth/reset/{token}", data={"password": "FirstNew1!"}) + assert first.status_code == 302 # success redirects to the login page + + replay = client.post(f"/auth/reset/{token}", data={"password": "AttackerPick1!"}) + assert replay.status_code == 200 + assert "already been used" in replay.get_data(as_text=True) + + login = client.post( + "/api/login", json={"email": "oncepage@example.com", "password": "FirstNew1!"} + ) + assert login.get_json()["status"] == "success" + + +def test_verify_link_replay_reads_as_verified(client): + """Mail scanners prefetch GET links and users re-click them; once the + account is verified, a replayed verify link should read as success, not + as a scary invalid-link page.""" + from routes._email import create_email_token + + _register(client, "reclick@example.com") + uid = _user_id("reclick@example.com") + token = create_email_token( + userid=uid, email="reclick@example.com", operation="email_address_verification" + ) + + for _ in range(2): + html = client.get(f"/auth/verify/{token}").get_data(as_text=True) + assert "not valid" not in html + + +def test_token_without_jti_rejected(client): + """Tokens minted before one-time-use stamping carry no jti; they can't be + tracked, so they are refused rather than replayable forever.""" + from datetime import UTC, datetime, timedelta + + import jwt as pyjwt + + from routes._email import EMAIL_TOKEN_AUDIENCE + from utils.flask_app import app + + _register(client, "nojti@example.com") + uid = _user_id("nojti@example.com") + token = pyjwt.encode( + { + "sub": { + "id": uid, + "email": "nojti@example.com", + "operation": "reset_password", + "org_id": -1, + }, + "exp": datetime.now(UTC) + timedelta(minutes=60), + "aud": EMAIL_TOKEN_AUDIENCE, + }, + app.config["JWT_SECRET_KEY"], + algorithm="HS256", + ) + resp = client.post("/api/reset_password", json={"token": token, "newPassword": "Sneaky1!"}) + assert resp.status_code == 400 diff --git a/back-end/tests/test_password_hashes.py b/back-end/tests/test_password_hashes.py new file mode 100644 index 0000000..76d055a --- /dev/null +++ b/back-end/tests/test_password_hashes.py @@ -0,0 +1,182 @@ +"""Legacy password-hash handling at the login endpoints. + +Accounts created on the old stack store werkzeug's legacy salted format +(``sha256$$``). werkzeug 3.x removed that method name, so a +raw ``check_password_hash`` raises ValueError and the login 500s instead of +logging the user in. The fix (utils/passwords.py) verifies legacy hashes with +the stdlib, never raises on malformed/None hashes, and transparently re-hashes +the account to pbkdf2:sha256 on the next successful login. Covers all three +login handlers: /auth/login, /api/login, /admin/login. +""" + +import hashlib +import hmac + +import pytest + +from tests.conftest import _db_reachable + +pytestmark = pytest.mark.skipif( + not _db_reachable(), reason="integration Postgres not reachable on localhost:5433" +) + + +def _legacy_sha256_hash(password, salt="abc123DEF456ghi7"): + """A stored hash exactly as old werkzeug (2.2.x ``method='sha256'``) wrote + it: ``sha256$salt$HMAC(key=salt, msg=password, sha256).hexdigest()``. + Verified against real werkzeug 2.2.3 output.""" + digest = hmac.new(salt.encode("utf-8"), password.encode("utf-8"), hashlib.sha256).hexdigest() + return f"sha256${salt}${digest}" + + +# --- unit: verify_password / needs_rehash (no DB) --------------------------- + + +def test_verify_password_legacy_roundtrip(): + from utils.passwords import verify_password + + stored = _legacy_sha256_hash("correct horse") + assert verify_password(stored, "correct horse") is True + assert verify_password(stored, "wrong") is False + + +def test_verify_password_current_pbkdf2(): + from werkzeug.security import generate_password_hash + + from utils.passwords import verify_password + + stored = generate_password_hash("Password123!", method="pbkdf2:sha256") + assert verify_password(stored, "Password123!") is True + assert verify_password(stored, "nope") is False + + +def test_verify_password_never_raises_on_bad_input(): + from utils.passwords import verify_password + + assert verify_password(None, "anything") is False + assert verify_password("", "anything") is False + assert verify_password("sha256$onlyonepart", "anything") is False + assert verify_password("total garbage", "anything") is False + assert verify_password("md5$salt$deadbeef", "anything") is False + assert verify_password(_legacy_sha256_hash("pw"), None) is False + + +def test_needs_rehash(): + from werkzeug.security import generate_password_hash + + from utils.passwords import needs_rehash + + assert needs_rehash(_legacy_sha256_hash("pw")) is True + assert needs_rehash(generate_password_hash("pw", method="pbkdf2:sha256")) is False + assert needs_rehash(None) is False + assert needs_rehash("") is False + + +# --- integration: the three login handlers ---------------------------------- + + +@pytest.fixture() +def client(db_session): + import routes # noqa: F401 + from utils.flask_app import app + + app.config["TESTING"] = True + with app.test_client() as c: + yield c + + +def _make_user(email, password_hash, **flags): + from database.models import user + from database.sessions import Session + + s = Session() + try: + u = user(email=email, password=password_hash, **flags) + s.add(u) + s.commit() + return u.id + finally: + s.close() + + +def _stored_hash(user_id): + from database.models import user + from database.sessions import Session + + s = Session() + try: + return s.query(user).filter(user.id == user_id).one().password + finally: + s.close() + + +def test_auth_login_migrates_legacy_hash(client): + uid = _make_user("legacy1@example.com", _legacy_sha256_hash("Password123!")) + + resp = client.post( + "/auth/login", data={"email": "legacy1@example.com", "password": "Password123!"} + ) + assert resp.status_code == 302 + assert resp.headers["Location"] == "/map" + assert any(c.startswith("token=") for c in resp.headers.getlist("Set-Cookie")) + + # The stored hash was transparently upgraded, and the password still works. + assert _stored_hash(uid).startswith("pbkdf2:") + resp = client.post( + "/auth/login", data={"email": "legacy1@example.com", "password": "Password123!"} + ) + assert resp.status_code == 302 + + +def test_api_login_migrates_legacy_hash(client): + uid = _make_user("legacy2@example.com", _legacy_sha256_hash("Password123!")) + + resp = client.post( + "/api/login", json={"email": "legacy2@example.com", "password": "Password123!"} + ) + assert resp.status_code == 200 + assert resp.get_json()["status"] == "success" + assert _stored_hash(uid).startswith("pbkdf2:") + + +def test_admin_login_migrates_legacy_hash(client): + uid = _make_user( + "legacyadmin@example.com", + _legacy_sha256_hash("Password123!"), + is_platform_admin=True, + ) + + resp = client.post( + "/admin/login", data={"email": "legacyadmin@example.com", "password": "Password123!"} + ) + assert resp.status_code == 302 + assert resp.headers["Location"] == "/admin/" + assert _stored_hash(uid).startswith("pbkdf2:") + + +def test_legacy_hash_wrong_password_rejected_not_migrated(client): + uid = _make_user("legacy3@example.com", _legacy_sha256_hash("Password123!")) + + resp = client.post("/auth/login", data={"email": "legacy3@example.com", "password": "wrong"}) + assert resp.status_code == 200 + assert "Invalid credentials" in resp.get_data(as_text=True) + assert _stored_hash(uid).startswith("sha256$") # untouched on failure + + +def test_malformed_hash_is_invalid_credentials_not_500(client): + # The reported bug: an unverifiable stored hash must read as bad + # credentials on every login door, never a server error. + _make_user("broken1@example.com", "sha256$onlyonepart") + _make_user("broken2@example.com", None) + + for email in ("broken1@example.com", "broken2@example.com"): + resp = client.post("/auth/login", data={"email": email, "password": "whatever"}) + assert resp.status_code == 200, email + assert "Invalid credentials" in resp.get_data(as_text=True) + + resp = client.post("/api/login", json={"email": email, "password": "whatever"}) + assert resp.status_code == 200, email + assert resp.get_json()["status"] == "error" + + resp = client.post("/admin/login", data={"email": email, "password": "whatever"}) + assert resp.status_code == 401, email diff --git a/back-end/tests/test_submissions_page.py b/back-end/tests/test_submissions_page.py index 1f18cbb..fdc34c8 100644 --- a/back-end/tests/test_submissions_page.py +++ b/back-end/tests/test_submissions_page.py @@ -504,3 +504,104 @@ def test_snapshots_never_leak_across_filings(client, db_session): # The lineage is explicit on the model, not inferred. snaps = db_session.query(folder_model).filter_by(type="export").all() assert {s.source_folder_id for s in snaps} == {folder_a.id, folder_b.id} + + +# ----------------------------------------- fabric debt tracks import success + + +def _seed_unimported_fabric(db_session, folder, task_status=None): + """A fabric file row whose async import never landed any fabric_data — + the state a filing is in while the worker runs, or after it failed.""" + from controllers.database_controller import file_ops + from database.models import celerytaskinfo + + f = file_ops.create_file( + filename="fabric.csv", + content=b"location_id\n", + folderid=folder.id, + filetype="fabric", + session=db_session, + ) + db_session.commit() + if task_status is not None: + db_session.add( + celerytaskinfo( + task_id="00000000-0000-0000-0000-000000000000", + status=task_status, + operation_type="Fabric", + operation_detail="Add the fabric", + user_email="debts@example.com", + folder_deadline=folder.deadline, + organization_id=folder.organization_id, + ) + ) + db_session.commit() + return f + + +def test_fabric_file_without_imported_rows_is_not_ready(db_session): + """The gate must key on the import having actually landed locations, not + on the file row (which is committed synchronously at upload, before — + and regardless of whether — the worker import succeeds).""" + from services import filing_service + + org = H.make_org(db_session) + user = H.make_user(db_session, org_id=org.id, email="unimported@example.com") + folder = H.make_folder(db_session, org.id) + _make_ready(db_session, folder) # everything else satisfied + + # Wipe the imported rows, keeping the file row: simulates a failed import. + from database.models import fabric_data + from database.models import file as file_model + + db_session.query(fabric_data).filter( + fabric_data.file_id.in_( + db_session.query(file_model.id).filter( + file_model.folder_id == folder.id, file_model.type == "fabric" + ) + ) + ).delete(synchronize_session=False) + db_session.commit() + + debts = filing_service.submission_debts(user.id, folder.id, db_session) + assert {d["id"] for d in debts} == {"fabric_import"} + + +def test_fabric_import_still_running_reads_as_in_progress(db_session): + from services import filing_service + + org = H.make_org(db_session) + user = H.make_user(db_session, org_id=org.id, email="importing@example.com") + folder = H.make_folder(db_session, org.id) + _seed_unimported_fabric(db_session, folder, task_status="PENDING") + + debts = filing_service.submission_debts(user.id, folder.id, db_session) + fabric_debt = next(d for d in debts if d["id"] == "fabric_import") + assert "in progress" in fabric_debt["label"].lower() + + +def test_fabric_import_failure_reads_as_failed(db_session): + from services import filing_service + + org = H.make_org(db_session) + user = H.make_user(db_session, org_id=org.id, email="failed@example.com") + folder = H.make_folder(db_session, org.id) + _seed_unimported_fabric(db_session, folder, task_status="FAILURE") + + debts = filing_service.submission_debts(user.id, folder.id, db_session) + fabric_debt = next(d for d in debts if d["id"] == "fabric_import") + assert "failed" in fabric_debt["label"].lower() + + +def test_fabric_import_debt_when_no_task_record(db_session): + """No task record at all (e.g. the row was lost): still not ready, with + the failed/retry wording rather than a false in-progress promise.""" + from services import filing_service + + org = H.make_org(db_session) + user = H.make_user(db_session, org_id=org.id, email="notask@example.com") + folder = H.make_folder(db_session, org.id) + _seed_unimported_fabric(db_session, folder, task_status=None) + + debts = filing_service.submission_debts(user.id, folder.id, db_session) + assert any(d["id"] == "fabric_import" for d in debts) diff --git a/back-end/tests/test_tiles_pipeline.py b/back-end/tests/test_tiles_pipeline.py index e2fb062..29cd5c2 100644 --- a/back-end/tests/test_tiles_pipeline.py +++ b/back-end/tests/test_tiles_pipeline.py @@ -267,3 +267,37 @@ def test_folder_copy_carries_rows_not_blob(db_session): assert len(copies) == 1 assert copies[0].tile_data is None assert len(tile_rows(db_session, copies[0].id)) == n_rows + + +# ------------------------------------------------- failure diagnosability + + +def test_run_tippecanoe_surfaces_stderr_on_failure(monkeypatch): + """A failed tippecanoe must raise with its real stderr in the message — + that string is what lands in celerytaskinfo.result, and a bare + "returned non-zero exit status 1" made a prod tile failure undiagnosable + without shell access to the worker.""" + import subprocess + + from controllers.database_controller import vt_ops + + def fake_run(command, **kwargs): + return subprocess.CompletedProcess( + args=command, returncode=1, stderr=b"/tmp/geom.XXKZPVzn: Too many open files\n" + ) + + monkeypatch.setattr(vt_ops.subprocess, "run", fake_run) + with pytest.raises(RuntimeError, match="Too many open files"): + vt_ops.run_tippecanoe("tippecanoe -o out.mbtiles data.geojson") + + +def test_run_tippecanoe_success_still_returns_zero(monkeypatch): + import subprocess + + from controllers.database_controller import vt_ops + + def fake_run(command, **kwargs): + return subprocess.CompletedProcess(args=command, returncode=0, stderr=b"1234 features\n") + + monkeypatch.setattr(vt_ops.subprocess, "run", fake_run) + assert vt_ops.run_tippecanoe("tippecanoe -o out.mbtiles data.geojson") == 0 diff --git a/back-end/utils/passwords.py b/back-end/utils/passwords.py new file mode 100644 index 0000000..db0fb4e --- /dev/null +++ b/back-end/utils/passwords.py @@ -0,0 +1,51 @@ +"""Non-throwing password verification for every login door. + +werkzeug 3.x removed the legacy salted hash methods (bare ``sha256``/``md5``), +so ``check_password_hash`` raises ValueError on hashes written by the old +stack (``sha256$$``) instead of returning False — which +turned every legacy-account login into a 500. Route all login checks through +``verify_password``: it verifies legacy hashes with the stdlib primitive the +old werkzeug used, and treats any malformed/None hash as bad credentials, +never a server error. Pair with ``needs_rehash`` to transparently upgrade an +account to pbkdf2 on its next successful login. +""" + +import hashlib +import hmac + +from werkzeug.security import check_password_hash + +# What new/migrated hashes are written with (user_ops uses the same scheme). +PREFERRED_METHOD = "pbkdf2:sha256" + + +def _verify_legacy_sha256(stored, password): + # Old werkzeug's salted simple hash for method='sha256': + # digest = HMAC(key=salt, msg=password, sha256).hexdigest() + # (validated against werkzeug 2.2.3, the version that wrote these hashes). + method, salt, digest = stored.split("$", 2) + if method != "sha256" or not salt or not digest: + return False + calc = hmac.new(salt.encode("utf-8"), password.encode("utf-8"), hashlib.sha256).hexdigest() + return hmac.compare_digest(calc, digest) + + +def verify_password(stored, password): + """True iff ``password`` matches the ``stored`` hash. Handles current + pbkdf2 hashes and legacy bare-sha256 hashes; returns False (rather than + raising) for None/empty/malformed hashes and non-string passwords.""" + if not stored or not isinstance(password, str): + return False + if stored.startswith("sha256$") and stored.count("$") == 2: + return _verify_legacy_sha256(stored, password) + try: + return check_password_hash(stored, password) + except (ValueError, TypeError): + # Unknown/removed method or malformed hash: bad credentials, not a 500. + return False + + +def needs_rehash(stored): + """True when a successfully-verified hash should be rewritten with + ``PREFERRED_METHOD`` (i.e. it still uses a legacy scheme).""" + return bool(stored) and not stored.startswith("pbkdf2:") diff --git a/deploy/data/docker-compose.yml b/deploy/data/docker-compose.yml index 0f81ad2..a1fe122 100644 --- a/deploy/data/docker-compose.yml +++ b/deploy/data/docker-compose.yml @@ -37,6 +37,14 @@ services: restart: unless-stopped cpus: ${WORKER_CPUS} mem_limit: ${WORKER_MEM_LIMIT} + # tippecanoe opens temp files scaled to the host's core count plus + # per-tile geometry files; on a many-core host a large tile build blows + # through Docker's default soft limit of 1024 and dies with EMFILE + # ("Too many open files"), deterministically. 65536 gives it headroom. + ulimits: + nofile: + soft: 65536 + hard: 65536 depends_on: - db - redis