diff --git a/src/bpp/newsfragments/import-list-if-liveops.feature.rst b/src/bpp/newsfragments/import-list-if-liveops.feature.rst new file mode 100644 index 000000000..5d85fc51c --- /dev/null +++ b/src/bpp/newsfragments/import-list-if-liveops.feature.rst @@ -0,0 +1,2 @@ +Import list IF przeniesiony z long_running na django-liveops: postęp +importu na żywo przez WebSocket, spójny z pozostałymi importerami. diff --git a/src/import_common/models.py b/src/import_common/models.py index 79686c895..6583f0e56 100644 --- a/src/import_common/models.py +++ b/src/import_common/models.py @@ -1,62 +1,3 @@ -from math import ceil - -from django.db import models - -from import_common.exceptions import XLSParseError -from import_common.util import XLSImportFile -from long_running.models import Operation -from long_running.notification_mixins import ASGINotificationMixin - - -class ImportOperation(ASGINotificationMixin, Operation): - plik_xls = models.FileField(upload_to="protected/import_common/") - - try_names = None - banned_names = None - min_points = None - - validation_form_class = None - - ignore_bad_rows = False - - def get_validation_form_class(self): - return self.validation_form_class - - def get_xls_import_file(self): - return XLSImportFile( - self.plik_xls.path, - try_names=self.try_names, - banned_names=self.banned_names, - min_points=self.min_points, - ) - - def perform(self): - x = self.get_xls_import_file() - - total = x.count() - five_percent = int(ceil(total / 50.0)) - form_class = self.get_validation_form_class() - - for no, elem in enumerate(x.data()): - cleaned_data = None - if form_class: - form = form_class(elem) - if not form.is_valid(): - if self.ignore_bad_rows: - continue - raise XLSParseError(elem, form, "wstępna weryfikacja danych") - - cleaned_data = form.cleaned_data - - self.import_single_row(xls_data=elem, cleaned_data=cleaned_data) - - if no % five_percent == 0: - self.send_progress(no * 100.0 / total) - - class Meta: - abstract = True - - class ImportRowMixin: def nr_arkusza(self): return self.dane_z_xls.get("__xls_loc_sheet__") diff --git a/src/import_common/util.py b/src/import_common/util.py index 241b6a962..4a6a607b2 100644 --- a/src/import_common/util.py +++ b/src/import_common/util.py @@ -45,9 +45,11 @@ def sprawdz_bombe_dekompresji(sciezka, max_rozpakowany=MAX_ROZMIAR_PO_DEKOMPRESJ f"potencjalna bomba dekompresyjna." ) + # openpyxl importujemy LOKALNIE (nie na poziomie modułu): ten moduł jest -# importowany eager przez import_common.models (XLSImportFile) już przy -# django.setup(), a openpyxl przez compat/numbers.py ciągnie całe numpy +# importowany eager przez modele importerów (np. import_list_if.models → +# XLSImportFile) już przy django.setup(), a openpyxl przez compat/numbers.py +# ciągnie całe numpy # (~tens MB RSS) do KAŻDEGO procesu — także web/ASGI, który nigdy nie czyta # xlsx. Dzięki PEP 563 (future annotations) adnotacje typu ``openpyxl.*`` # w sygnaturach poniżej są tylko łańcuchami i nie wymagają openpyxl w diff --git a/src/import_list_if/migrations/0005_liveops.py b/src/import_list_if/migrations/0005_liveops.py new file mode 100644 index 000000000..0fcbdf213 --- /dev/null +++ b/src/import_list_if/migrations/0005_liveops.py @@ -0,0 +1,82 @@ +# Generated by Django 5.2.16 on 2026-07-16 18:57 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("import_list_if", "0004_alter_importlistif_plik_xls"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AlterModelOptions( + name="importlistif", + options={"ordering": ["-created_on"]}, + ), + migrations.RemoveField( + model_name="importlistif", + name="last_updated_on", + ), + migrations.AddField( + model_name="importlistif", + name="cancel_requested", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="importlistif", + name="cancelled", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="importlistif", + name="current_stage", + field=models.IntegerField(default=-1), + ), + migrations.AddField( + model_name="importlistif", + name="language", + field=models.CharField(blank=True, default="", max_length=20), + ), + migrations.AddField( + model_name="importlistif", + name="log", + field=models.JSONField(default=list), + ), + migrations.AddField( + model_name="importlistif", + name="log_seq", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="importlistif", + name="percent", + field=models.PositiveSmallIntegerField(default=0), + ), + migrations.AddField( + model_name="importlistif", + name="result_context", + field=models.JSONField(blank=True, null=True), + ), + migrations.AddField( + model_name="importlistif", + name="stage_states", + field=models.JSONField(default=dict), + ), + migrations.AddField( + model_name="importlistif", + name="status_text", + field=models.CharField(blank=True, default="", max_length=255), + ), + migrations.AlterField( + model_name="importlistif", + name="owner", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ] diff --git a/src/import_list_if/models.py b/src/import_list_if/models.py index 436b81027..2678acde9 100644 --- a/src/import_list_if/models.py +++ b/src/import_list_if/models.py @@ -1,6 +1,7 @@ from django import forms -from django.db import models +from django.db import models, transaction from django.db.models import JSONField +from liveops.models import LiveOperation from bpp.models import ( IF_DECIMAL_PLACES, @@ -10,7 +11,9 @@ Zrodlo, ) from import_common.core import matchuj_zrodlo -from import_common.models import ImportOperation, ImportRowMixin +from import_common.exceptions import XLSParseError +from import_common.models import ImportRowMixin +from import_common.util import XLSImportFile class ImportListRowValidationForm(forms.Form): @@ -20,14 +23,65 @@ class ImportListRowValidationForm(forms.Form): ) -class ImportListIf(ImportOperation): +class ImportListIf(LiveOperation): rok = models.PositiveSmallIntegerField() + plik_xls = models.FileField(upload_to="protected/import_common/") try_names = ["full_journal_title", "journal_impact_factor"] banned_names = [] min_points = 2 validation_form_class = ImportListRowValidationForm + ignore_bad_rows = False + + def get_validation_form_class(self): + return self.validation_form_class + + def get_xls_import_file(self): + return XLSImportFile( + self.plik_xls.path, + try_names=self.try_names, + banned_names=self.banned_names, + min_points=self.min_points, + ) + + def run(self, p): + # Punkt wejścia liveops (dawniej ImportOperation.perform()). Owijamy + # ciało w transaction.atomic() dla parytetu z legacy task_perform: + # OperationCancelled / XLSParseError cofa (rollback) już zapisane + # wiersze → import jest all-or-nothing. p.result() jest POZA blokiem, + # więc panel wyniku dostaje policzone (zacommitowane) wiersze. + with transaction.atomic(): + x = self.get_xls_import_file() + total = x.count() + form_class = self.get_validation_form_class() + + for no, elem in enumerate(x.data()): + p.check_cancelled() + + cleaned_data = None + if form_class: + form = form_class(elem) + if not form.is_valid(): + if self.ignore_bad_rows: + continue + raise XLSParseError(elem, form, "wstępna weryfikacja danych") + cleaned_data = form.cleaned_data + + self.import_single_row(xls_data=elem, cleaned_data=cleaned_data) + + if total: + p.percent(int((no + 1) * 100 / total)) + + wiersze = self.get_details_set() + p.result( + { + "total": total, + "zintegrowano": wiersze.filter(zintegrowano=True).count(), + "niedopasowane": wiersze.filter(zrodlo__isnull=True).count(), + "rok": self.rok, + } + ) def import_single_row(self, xls_data, cleaned_data): zrodlo = matchuj_zrodlo(cleaned_data.get("full_journal_title")) @@ -57,11 +111,16 @@ def import_single_row(self, xls_data, cleaned_data): res.save() - def on_reset(self): + def on_restart(self): + # Hook liveops (RestartView) — odpowiednik dawnego on_reset(). Kasuje + # wiersze poprzedniego przebiegu, by restart zaczynał od czysta. self.get_details_set().delete() def get_details_set(self): - return self.importlistifrow_set.all().select_related("zrodlo") + # order_by("id") = kolejność wstawiania = kolejność wierszy w pliku; + # deterministyczna paginacja tabeli wyników (ImportListIfRow nie ma + # Meta.ordering, więc bez tego paginator ostrzega o niestabilności). + return self.importlistifrow_set.all().select_related("zrodlo").order_by("id") class ImportListIfRow(ImportRowMixin, models.Model): diff --git a/src/import_list_if/templates/import_list_if/import_list_if.html b/src/import_list_if/templates/import_list_if/import_list_if.html new file mode 100644 index 000000000..4bd35d089 --- /dev/null +++ b/src/import_list_if/templates/import_list_if/import_list_if.html @@ -0,0 +1,36 @@ +{% extends "base.html" %} +{% load static liveops %} + +{% block extratitle %} + Import list IF - {{ object.plik_xls.name }} +{% endblock %} + +{% block breadcrumbs %} + {{ block.super }} +
  • import list IF
    • +
  • import {{ object.plik_xls.name }}
    • +{% endblock %} + +{% block content %} +

    Import list IF — przetwarzanie

    + + {# Region live-operacji: kontener z data-liveop-channel/token + strefy #} + {# op-status/op-progress/op-log/op-result (OOB-swap z workera). Panel #} + {# wyniku (z linkiem do tabeli wyników) pojawia się w #op-result dopiero #} + {# po zakończeniu importu — patrz import_list_if_result.html. #} + {# hx-headers wstrzykuje token CSRF nagłówkiem X-CSRFToken: przyciski #} + {# Anuluj/Ponów liveops to gołe przyciski htmx poza formularzem, a BPP ma #} + {# CSRF_COOKIE_HTTPONLY=True, więc liveops.js nie odczyta tokenu z ciasteczka #} + {# (getCookie zwraca pusty łańcuch) → POST bez tokenu = 403 CSRF token missing. #} + {# Wrapper nie jest podmieniany przy OOB-swapie regionów op-*, więc token #} + {# dziedziczą też przyciski dorenderowane po odświeżeniu przez WebSocket. #} +
    + {% live_operation object %} +
    + + {# Kolejność ładowania jest istotna: htmx → klient channels_broadcast → #} + {# liveops.js (patchuje channelsBroadcast.addMessage i inicjuje socket). #} + + + +{% endblock %} diff --git a/src/import_list_if/templates/import_list_if/import_list_if_result.html b/src/import_list_if/templates/import_list_if/import_list_if_result.html new file mode 100644 index 000000000..55c673176 --- /dev/null +++ b/src/import_list_if/templates/import_list_if/import_list_if_result.html @@ -0,0 +1,20 @@ +{# Fragment wyniku (liveops result_template_name). Renderowany po p.result() #} +{# jako OOB-swap #op-result oraz przy wejściu na zakończoną operację. #} +{# Kontekst: operation + klucze z result_context (total, zintegrowano, #} +{# niedopasowane, rok). #} +
    +

    + + Import zakończony +

    +

    + Przetworzono wierszy: {{ total }}, + zintegrowano: {{ zintegrowano }}, + niedopasowanych źródeł: {{ niedopasowane }}{% if rok %} + (rok {{ rok }}){% endif %}. +

    + + Zobacz pełne wyniki importu + +
    diff --git a/src/import_list_if/templates/import_list_if/importlistif_detail.html b/src/import_list_if/templates/import_list_if/importlistif_detail.html deleted file mode 100644 index b12b084d3..000000000 --- a/src/import_list_if/templates/import_list_if/importlistif_detail.html +++ /dev/null @@ -1,27 +0,0 @@ -{% extends "base.html" %}{% load render_table from django_tables2 %} - -{% block extratitle %} - Import list IF - szczegóły {{ object.plik_xls.name }} -{% endblock %} - -{% block breadcrumbs %} - {{ block.super }} -
  • import list IF
    • -
  • import {{ object.plik_xls.name }}
    • -{% endblock %} - - -{% block content %} -

    Import danych {{ object.plik_xls.name }}

    - - - -{% endblock %} diff --git a/src/import_list_if/templates/import_list_if/importlistif_form.html b/src/import_list_if/templates/import_list_if/importlistif_form.html index 581398387..168c32b5c 100644 --- a/src/import_list_if/templates/import_list_if/importlistif_form.html +++ b/src/import_list_if/templates/import_list_if/importlistif_form.html @@ -1,4 +1,4 @@ -{% extends "base.html" %}{% load render_table from django_tables2 %} +{% extends "base.html" %} {% block extratitle %} Import list IF - nowy diff --git a/src/import_list_if/templates/import_list_if/importlistif_list.html b/src/import_list_if/templates/import_list_if/importlistif_list.html index 64cef804c..464a812e3 100644 --- a/src/import_list_if/templates/import_list_if/importlistif_list.html +++ b/src/import_list_if/templates/import_list_if/importlistif_list.html @@ -1,4 +1,4 @@ -{% extends "base.html" %}{% load render_table from django_tables2 %} +{% extends "base.html" %} {% block extratitle %} Import list IF @@ -6,7 +6,7 @@ {% block breadcrumbs %} {{ block.super }} -
  • import list IF
    • +
  • import list IF
    • {% endblock %} {% block content %}{% load static %} @@ -24,7 +24,7 @@

    Ostatnio importowane dane: