From 1d3be286fe747f718872803cd62ecd4d235b08e2 Mon Sep 17 00:00:00 2001 From: voidReq <100312344+voidReq@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:38:30 -0400 Subject: [PATCH 1/2] Faster screens: keep the opened sheet, cache Review's list, instant button feedback - sheet.py: the opened spreadsheet and its tabs are kept 5 minutes (one request per read instead of three); opened again on a new login, an expiring token, a refused request, a renamed or deleted tab - Apply: answered from the rows read last; opening it re-reads both tabs in the background; sheet_version moves only when a read finds a change; each tab draws as soon as its answer comes - Review: the list and the taste model are kept until data/, settings or the sheet rows change, and made ahead at start and after decisions - Buttons show busy at once and ignore double clicks; row-menu sheet changes show a working toast; switching screens shows Loading at once Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 5 +- docs/ARCHITECTURE.md | 6 +- docs/DEPLOY.md | 6 +- docs/GUI.md | 25 +++- docs/PIPELINE.md | 3 +- src/nuauto/demo.py | 7 +- src/nuauto/gui.py | 231 +++++++++++++++++++++++++--------- src/nuauto/gui_static/app.css | 2 + src/nuauto/gui_static/app.js | 88 +++++++++---- src/nuauto/intern.py | 10 +- src/nuauto/jobs.py | 32 ++++- src/nuauto/sheet.py | 100 +++++++++++++-- tests/test_gui.py | 20 ++- tests/test_intern.py | 15 ++- tests/test_sheet.py | 96 +++++++++++++- 15 files changed, 521 insertions(+), 125 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index dc1ba6f..af2b9d9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -155,8 +155,9 @@ Status values: Proposed, Approved, Applied, Failed, Needs Human, Accepted, Rejec leave the NUworks tab (they count toward the caps); an Applied Other job may move to NUworks (then it counts there; my call 2026-10-09). Unresolved Submit clicks are never moved or removed. Google access is OAuth only, never service accounts (docs/DEPLOY.md, GOOGLE SHEETS LOGIN). Google allows about 60 reads a -minute: every client waits and retries on "too many requests" (gspread.BackOffHTTPClient), and the internship Review reuses -the Other jobs rows for 30 s (each change NUauto makes forgets them). +minute: every client waits and retries on "too many requests" (sheet.HTTPClient, gspread's BackOffHTTPClient); the opened +sheet is kept 5 minutes (sheet.OPEN; rows are always read fresh), and Apply and the internship Review reuse the Other jobs +rows for 30 s (each change NUauto makes forgets them; docs/GUI.md, SPEED). BROWSER (Playwright, Firefox) - Deterministic script, not an AI clicking around freely. Slow, human-like pacing. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6b66f20..e85ae95 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -13,7 +13,7 @@ MODULES (src/nuauto/) app: bundle paths, app-data STATE_DIR, local mode, Playwright's browser cache), self_cmd() / self_exe() (how NUauto runs itself). No logic beyond that. sheet.py the Google Sheet. client() / open_worksheet(interactive) (interactive=False never opens a - login page: NotLoggedIn), read_rows(), check_limits() (MAX_PER_WEEK/MAX_TOTAL), set_status(), + login page: NotLoggedIn; the opened sheet and tabs kept OPEN_FOR s: OPEN, forget(), HTTPClient), read_rows(), check_limits() (MAX_PER_WEEK/MAX_TOTAL), set_status(), mark_submit_started()/resolve_submit(), mark_applied_by_hand(), mark_site_done(), append_note(), add_proposed(), unapprove() (GUI Undo), create_sheet() (Sheets API), google_login(). jobs.py the job pool + viewers. PREFS (local_config "preferences" over DEFAULTS), RULES, set_prefs(), @@ -190,7 +190,9 @@ changes the sheet). test_browser.py covers the domain lock and the profile lock. company-site agent's guard. test_mail.py covers the email codes and links (Gmail's message format, which mail counts, the agent's `email` command). test_insights.py covers the Insights numbers (pay parsing, folding, no guessing). test_intern.py covers the internship pool and postings.py (job-site links, parsing, the network guard on a local server, fetch tries, rules, -Review queue, a whole update with Claude stood in for). test_deploy.py runs deploy.py on a throwaway git repo (copy, web restart only when needed, undo on failure). test_demo.py runs the real apply.py on demo mode's fake pages (stops, cap, re-login). +Review queue (the taste model trains again only when ratings or a job's text change), a whole update with Claude +stood in for). test_sheet.py also covers the kept sheet (sheet.OPEN: opened once, opened again on a new login, an +expiring token, a refused request, a renamed tab). test_deploy.py runs deploy.py on a throwaway git repo (copy, web restart only when needed, undo on failure). test_demo.py runs the real apply.py on demo mode's fake pages (stops, cap, re-login). test_gui.py covers the GUI server's security and flows; test_homelab_app.py the same app as the homelab serves it (demo `--remote`: pairing, refusals, sessions, queue, devices, a phone-width browser); test_link.py the laptop's link to it (pairing, calls, the /api/homelab/ routes, an Apply run there, unpairing; then `--demo --linked` in a browser); tests/screens.py (not a test: diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 0910252..1a04eff 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -138,7 +138,11 @@ GOOGLE SHEETS LOGIN (gspread + OAuth) - Testing mode -> a login lasts 7 days. Re-login on the laptop: `nuauto login google`; it pushes the new token to the homelab. Discord warns the day before; laptop commands re-open the Google login by themselves when it has expired. -- Open the sheet with open_by_key(SHEET_ID) so only the Sheets API is needed. +- Open the sheet with open_by_key(SHEET_ID) so only the Sheets API is needed. Each process keeps the opened sheet + and its tabs 5 minutes (sheet.OPEN, OPEN_FOR; rows are always read fresh), so a read is one request: opened again + when the sheet ID or the login files change, when the access token is about to expire (an expired login is still + NotLoggedIn), and after a request Google refuses (sheet.HTTPClient, gspread's BackOffHTTPClient: it also waits and + retries on "too many requests"). - Optional: the assistant's email codes (mail.py) need the Gmail API enabled in the same project (console.cloud.google.com/apis/library/gmail.googleapis.com), then `nuauto login gmail` on the laptop: Google's read-only Gmail permission, a token of its own (local/gmail_token.json), so the Sheets token never reads mail. It diff --git a/docs/GUI.md b/docs/GUI.md index 2bc9fd7..b86f731 100644 --- a/docs/GUI.md +++ b/docs/GUI.md @@ -135,9 +135,12 @@ SCREENS and where you answer it (its screen; a laptop session: its terminal window). GET /api/jobs?tab= is one tab's every row in its sections with what Move / Remove allow (gui.jobs_tab); /api/apply, /api/company, /api/other, /api/sheet stay (the laptop's link reads the homelab's /api/company). - The lists follow the sheet by themselves: /api/state carries rows_at (when the sheet was last read), the - server reads it again after every finished row and run and when it is 30 s old, and the open screen - reloads its lists when rows_at changes (no closing and reopening). "At most" keeps what you typed. The menu's + The lists follow the sheet by themselves: /api/state carries sheet_version (+1 each time a read finds + either tab changed), the server reads both tabs again after every finished row and run, when an + assistant session starts or ends, and when they are 30 s old, and the open screen reloads its lists when + sheet_version changes (no closing and reopening; each /api/jobs answer carries the version its rows are as + new as, so the screen's own reload is not followed by a second one that would close an open menu). Opening the screen is answered from the rows read last + and reads both tabs again behind it (SPEED). Each tab is drawn as soon as its own answer comes. "At most" keeps what you typed. The menu's count is Approved + the assistant's rows + retries. #/sheet (the old Sheet screen) opens Apply. Insights NUworks co-ops / " internships" switch when internships are on (2026-10-16; remembered for the session): each its own pool, applications and sheet tab (insights.collect / collect_intern: the internship @@ -224,6 +227,20 @@ SETUP WIZARD (onboard.py + gui_static/setup.js; each step checks itself) homelab optional, a source install only: the same Homelab card as Settings (done when NUauto runs with one; then its NUauto on your homelab part shows there too: one step for all of it) +SPEED (2026-10-22, my request: "the apply page takes quite a while to load") +- Google Sheets: the opened spreadsheet and its tabs are kept 5 minutes (sheet.OPEN; a read is one trip to Google, + about 0.25 s, not three); rows are always read fresh. Opened again on a new Google login, an access token about to + expire, or a refused request (a tab renamed or deleted: read_rows opens it again once). docs/DEPLOY.md. +- Apply: answered from the rows read last (the NUworks tab's: up to 60 s; the Other jobs tab's: up to 30 s, + gui.other_rows); opening the screen (recheck=1) reads both again in the background, and a change reloads the lists. +- Review: the list (pool, taste model, order) is kept until data/, your settings or the sheet's rows change + (gui.review_lists, keyed by every data/ file's size and time: a few ms); searches and filters only narrow it. The + taste model trains again only when your ratings or a job's text change (jobs.taste_for). The lists are made ahead at + start and a few seconds after your last decision (warm_review), so the first open and the next search are instant. +- Buttons: one whose click starts something slower shows it at once (aria-busy) and takes no second click until it + is done (busyWhile); sheet changes from a row's menu show "Updating the sheet…" until the answer comes. Switching + screens shows Loading… at once; a screen you already left never takes over. + HEALTH (health.py; the same checks `nuauto doctor` prints, `nuauto doctor --json`) check how GUI runs it files settings saved, local/ 700 and secrets 600 (Fix permissions) every minute @@ -335,7 +352,7 @@ VIEW MODE (`nuauto gui --view`; gui.VIEW; added 2026-10-09 so an agent can look (a newer push to a pull request stops its older run). API (JSON; all need the cookie; POSTs need X-NUauto: 1) - GET /api/state counts, week, to-do, last scan, setup_needed, the current task; internships {on, term, + GET /api/state counts, week, to-do, last scan, setup_needed, the current task, sheet_version; internships {on, term, review: how many wait for Review}; where (laptop | homelab), laptop_only {action: why} (the homelab's page), remote_app (the laptop: the homelab's page address, when it has one); todo: the assistants that wait for you first ({"kind": "assistant", "status", "text", "screen": its diff --git a/docs/PIPELINE.md b/docs/PIPELINE.md index 8ece4ea..ce1c3c0 100644 --- a/docs/PIPELINE.md +++ b/docs/PIPELINE.md @@ -80,7 +80,8 @@ RANKING (my settings: the engineering field) MY RATINGS (taste model) - `nuauto rate` (y/n, nothing goes in the sheet) -> data/ratings.json, which the laptop owns and pushes to the homelab. They train a TF-IDF model that reorders the pool (half rank, - half my taste) once there are 5 yes + 5 no. + half my taste) once there are 5 yes + 5 no. Trained again only when my ratings or a job's text change + (jobs.taste_for keeps the last few; training takes about a second). - Rating order rotates categories (security, embedded, hardware, systems, robotics/test, web/full-stack, software, data/ML, IT, other, front-end) so every category gets rated; once the model trains, it also mixes in the jobs it is least sure about. diff --git a/src/nuauto/demo.py b/src/nuauto/demo.py index ec162dd..833337a 100644 --- a/src/nuauto/demo.py +++ b/src/nuauto/demo.py @@ -199,8 +199,11 @@ def _locked(self): return fd def _load(self): - with open(self.path) as f: - return json.load(f) + try: + with open(self.path) as f: + return json.load(f) + except FileNotFoundError: # the tab is gone (a reset, a test): what Google answers for a deleted tab + raise gspread.exceptions.APIError(FakeResponse({"error": {"code": 400, "message": f"demo: no tab {self.title!r}"}})) def get_all_values(self): rows = [list(r) for r in self._load()["rows"]] diff --git a/src/nuauto/gui.py b/src/nuauto/gui.py index b90116e..095b8c3 100644 --- a/src/nuauto/gui.py +++ b/src/nuauto/gui.py @@ -213,8 +213,12 @@ def __init__(self): self.rows, self.rows_at, self.rows_error, self.rows_try_at = None, 0, None, 0 self.queue = {} # job id -> pool entry, from the last review queue self.other_urls = None # the Other jobs tab's links (not Proposed), as last read: the internship count - self.other_at = 0 # when they were last asked for - self.other_cache = None # (when, rows) of the Other jobs tab, for the internship Review (gui.other_rows) + self.other_cache = None # (when, rows) of the Other jobs tab (gui.other_rows) + self.other_last, self.other_exists = None, None # the tab's rows as last read (kept by other_changed), is there one + self.other_try_at, self.other_reading = 0, False + self.other_gen = 0 # +1 on each change NUauto makes to the tab: a read begun before it is not kept + self.sheet_version = 0 # +1 each time a read finds either tab changed: the open screen reloads its lists then + self.seen_assistants = None # the assistant sessions state() last saw (one ending may have changed its row) self.last_request = now() self.stopping = False self.window_kind = "none" # "mac" / "gtk" (NUauto's own window), "app" / "tab" (a browser), "none" @@ -291,14 +295,32 @@ def refresh_after_row(self): def refresh_soon(self, max_age=0): """Read the sheet again in the background when the rows are older than max_age seconds, so the lists and counts follow changes made elsewhere (a row finished in a run, an assistant in a terminal, the sheet itself). A failed - read is not tried again for 30 s.""" + read is not tried again for max_age s (0: now, unless a read is under way). The Other jobs tab too: its screens + use the rows read last (other_rows).""" + self.refresh_other_soon(max_age) if self.rows is None or "sheet" in self.running_groups: return - if now() - max(self.rows_at, self.rows_try_at) < max(max_age, 1): + if now() - max(self.rows_at, self.rows_try_at) < max_age: return self.rows_try_at = now() threading.Thread(target=self.refresh_rows, kwargs={"force": True}, daemon=True).start() + def refresh_other_soon(self, max_age=0): + with self.lock: + read_at = self.other_cache[0] if self.other_cache else 0 + if self.other_reading or now() - max(read_at, self.other_try_at) < max_age: + return + self.other_reading, self.other_try_at = True, now() + + def read(): + try: + other_rows() + except Exception: + pass + finally: + self.other_reading = False + threading.Thread(target=read, daemon=True).start() + # ---- health GROUPS = { # group -> (check names, seconds between runs, first run after) @@ -328,6 +350,8 @@ def run_group(self, group): checks, rows = health.sheet_and_limits() with self.lock: if rows is not None: + if rows != self.rows: + self.sheet_version += 1 self.rows, self.rows_at, self.rows_error = rows, now(), None else: self.rows_error = checks[0].detail @@ -423,7 +447,13 @@ def monitor(self): groups = [g for g in self.GROUPS if group_on(g)] for group in groups: self.next_run[group] = start + self.GROUPS[group][2] + warmed = False while not self.stopping: + if not warmed and self.rows is not None and now() - start > 5: # Review's lists, made ahead (1-5 s each) + warmed = True + from nuauto import intern + threading.Thread(target=warm_review, daemon=True, + args=([("approve", "nuworks")] + ([("approve", "intern")] if intern.ENABLED else []),)).start() for group in groups: if now() >= self.next_run.get(group, 0) and group not in self.running_groups: self.next_run[group] = now() + self.GROUPS[group][1] @@ -572,14 +602,11 @@ def review_counts(rows): def intern_count(): """Cheap count for the menu: internships in Review (the pool and the ones that could not be read) that are not in - the Other jobs tab and not rated no. The tab's links are read in the background (at start, then every 10 minutes; - each internship Review and decision updates them too).""" + the Other jobs tab and not rated no. The tab's links are read in the background (APP.refresh_soon; each internship + Review and decision updates them too).""" from nuauto import intern if not intern.ENABLED: return 0 - if now() - APP.other_at > (600 if APP.other_urls is not None else 60): - APP.other_at = now() - threading.Thread(target=read_other_quietly, daemon=True).start() ratings, seen = jobs.load("ratings.json", {}), APP.other_urls or set() return sum(1 for r in intern.load("pool.json", []) + intern.load("unread.json", []) if r["url"] not in seen and ratings.get(r["id"], {}).get("label") != 0) @@ -614,6 +641,16 @@ def state(): sheet_id = cfg.get("sheet_id") or "" task = APP.task from nuauto import intern + # which assistant sessions run now and what they want (the open screens reload their lists when this changes) + sessions = (",".join(f"{s.get('tab')}-{s.get('row')}:{s.get('slot')}:{(s.get('attention') or {}).get('kind', '')}" + for s in assistants()["sessions"]) + + "|" + ",".join(f"{s.get('tab')}-{s.get('row')}:{(s.get('attention') or {}).get('kind', '')}" + for s in homelab_assistants())) + if sessions != APP.seen_assistants: # one that ended may have changed its row: the lists read the sheet again + if APP.seen_assistants is not None: + other_changed() + APP.refresh_soon() + APP.seen_assistants = sessions return { "demo": config.DEMO, "view": VIEW, "version": __import__("nuauto").__version__, # which machine this page runs on: "laptop" (nuauto gui) or "homelab" (its app, serve_remote), and what only the @@ -623,13 +660,9 @@ def state(): "homelab_link": link_state(), # (laptop) this app's link to it: start there, its sessions, its runs "commit": code_commit(), # which code this machine runs (both pages show it) "internships": {"on": intern.ENABLED, "term": intern.TERM, "review": intern_count()}, - # which assistant sessions run now and what they want (the open screens reload their lists when this changes) - "assistants": ",".join(f"{s.get('tab')}-{s.get('row')}:{s.get('slot')}:{(s.get('attention') or {}).get('kind', '')}" - for s in assistants()["sessions"]) - + "|" + ",".join(f"{s.get('tab')}-{s.get('row')}:{(s.get('attention') or {}).get('kind', '')}" - for s in homelab_assistants()), + "assistants": sessions, "mode": "homelab" if config.HAS_SERVER else "local", "setup_needed": setup_needed(), - "rows_loaded": rows is not None, "rows_at": APP.rows_at, "rows_error": APP.rows_error, "week": week_info(rows), + "rows_loaded": rows is not None, "rows_at": APP.rows_at, "sheet_version": APP.sheet_version, "rows_error": APP.rows_error, "week": week_info(rows), "counts": {"review": len(todo), "approved": len(approved), "company": len(agent) + len(retry), "site": len(site), "apply": len(approved) + len(agent) + len(retry), # the Apply menu item: its rows + Company sites "needs_human": len(agent) + len(other)}, @@ -696,7 +729,8 @@ def todo_list(site, agent, soon): return waiting + rest -OTHER_ROWS_FOR = 30 # seconds the internship Review reuses the Other jobs tab's rows (it asks on every search) +OTHER_ROWS_FOR = 30 # seconds the Apply screen and the internship Review reuse the Other jobs tab's rows (Review asks on +# every search); /api/state reads them again in the background when they are that old def other_rows(fresh=True): @@ -704,45 +738,118 @@ def other_rows(fresh=True): the rows read in the last OTHER_ROWS_FOR seconds will do (each read costs Google quota: about 60 a minute; every change NUauto makes to the tab forgets them).""" from nuauto import intern - if not fresh and APP.other_cache and now() - APP.other_cache[0] < OTHER_ROWS_FOR: - return APP.other_cache[1] + cached, gen = APP.other_cache, APP.other_gen + if not fresh and cached and now() - cached[0] < OTHER_ROWS_FOR: + return cached[1] try: - rows = sheet.read_rows(worksheet(other=True)) + rows, exists = read_tab(other=True), True except NoOtherTab: - rows = [] - APP.other_urls = intern.sheet_urls(rows) - APP.other_cache = (now(), rows) + rows, exists = [], False + with APP.lock: + if gen != APP.other_gen: # NUauto changed the tab while this was being read: the next ask reads it again + return rows + if rows != APP.other_last: + APP.sheet_version += 1 + APP.other_last, APP.other_exists = rows, exists + APP.other_urls = intern.sheet_urls(rows) + APP.other_cache = (now(), rows) return rows def other_changed(): - """NUauto changed the Other jobs tab: the next internship Review reads it again.""" - APP.other_cache = None + """NUauto changed the Other jobs tab (or may have): the next Apply screen or internship Review reads it again.""" + with APP.lock: + APP.other_gen += 1 + APP.other_cache = None + +_review = {} # (mode, source) -> (what it was made from, todo, urgent, details) +_review_locks = {} -def read_other_quietly(): - """other_rows in the background (for the count): a sheet that can't be read now is tried again later.""" + +def data_key(): + """What Review's queue is made from, cheaply (a few ms): every file in data/ (its size and time: ratings, scores, + the pool, each job's details) and your settings.""" + out = [] + for root, dirs, files in os.walk(config.DATA_DIR): + dirs.sort() + for f in sorted(files): + try: + st = os.stat(os.path.join(root, f)) + except OSError: + continue + out.append((root, f, st.st_mtime_ns, st.st_size)) try: - other_rows() - except Exception: + st = os.stat(config.LOCAL_CONFIG_PATH) + out.append((st.st_mtime_ns, st.st_size)) + except OSError: pass + return hash(tuple(out)) -def review(mode, q="", category="", source="nuworks"): - """The review queue; q (words or "quoted phrases", all must appear) and category narrow what is shown. - Decisions work on any job of the whole queue. source "intern": the internships (intern.py), whose Approve adds a - row to the Other jobs tab.""" +def row_key(rows): + return tuple((r.number, r.url, r.status) for r in rows or []) + + +def review_lists(mode, source="nuworks"): + """(todo, urgent, details) of the review queue, kept until what it is made from changes (training the taste model and + building the pool take 1-5 s; Review asks on every load and search).""" rows = need_rows() - ratings = jobs.load("ratings.json", {}) if source == "intern": from nuauto import intern if not intern.ENABLED: raise Refused('Internships are off (add "internships": {} to local/local_config.json).') - todo, urgent = intern.review_queue(mode, ratings, other_rows(fresh=False), rows) - details = {r["id"]: intern.details_for(r) for r in todo} - else: - details = jobs.load_details() - todo, urgent = jobs.review_queue(mode, details, ratings, rows) + other = other_rows(fresh=False) + with _review_locks.setdefault((mode, source), threading.Lock()): # (a warm-up under way: wait for it, don't redo it) + key = (data_key(), date.today(), row_key(rows), row_key(other) if source == "intern" else None) # (before reading) + kept = _review.get((mode, source)) + if kept and kept[0] == key: + return kept[1:] + ratings = jobs.load("ratings.json", {}) + if source == "intern": + todo, urgent = intern.review_queue(mode, ratings, other, rows) + details = {r["id"]: intern.details_for(r) for r in todo} + else: + details = jobs.load_details() + todo, urgent = jobs.review_queue(mode, details, ratings, rows) + _review[(mode, source)] = (key, todo, urgent, details) + return todo, urgent, details + + +def warm_review(pairs): + """Make the review lists ahead (in the background): at start, and a few seconds after your last decision.""" + for mode, source in pairs: + try: + review_lists(mode, source) + except Exception: + pass + + +_warm = {"timer": None, "last": ("approve", "nuworks")} # last: the list Review showed last + + +def warm_review_soon(): + """After a decision: the list Review shows is made again once you pause (WARM_AFTER s), so the next load or search + is instant. Only while the window's server runs.""" + if SERVER is None: + return + if _warm["timer"]: + _warm["timer"].cancel() + _warm["timer"] = threading.Timer(WARM_AFTER, warm_review, args=([_warm["last"]],)) + _warm["timer"].daemon = True + _warm["timer"].start() + + +WARM_AFTER = 3 + + +def review(mode, q="", category="", source="nuworks"): + """The review queue; q (words or "quoted phrases", all must appear) and category narrow what is shown. + Decisions work on any job of the whole queue. source "intern": the internships (intern.py), whose Approve adds a + row to the Other jobs tab.""" + todo, urgent, details = review_lists(mode, source) + _warm["last"] = (mode, source) + ratings = jobs.load("ratings.json", {}) APP.queue = {r["id"]: r for r in todo} counts = {} for r in todo: @@ -807,6 +914,15 @@ def worksheet(other=False): raise Refused(str(e)) +def read_tab(other=False): + """worksheet's rows. A tab that went away since it was opened (sheet.read_rows opens it again) is NoOtherTab too.""" + ws = worksheet(other) + try: + return sheet.read_rows(ws) + except sheet.NoOtherTab as e: + raise NoOtherTab(str(e)) + + def decide(body): """Review answers. approve: the job goes in the sheet as Approved now (and counts as a yes for the ranking); yes / no: a rating only (no is skipped by Review from now on). Returns the new row for approve.""" @@ -841,6 +957,7 @@ def decide(body): APP.refresh_rows(force=True) out["row"] = next((x.number for x in APP.rows or [] if x.url == jobs.job_url(job_id)), None) jobs.set_rating(job_id, {"label": int(decision != "no"), "date": date.today().isoformat()}) + warm_review_soon() return out @@ -867,6 +984,7 @@ def undo(body): APP.refresh_rows(force=True) jobs.set_rating(job_id, {"label": previous["label"], "date": str(previous.get("date", ""))[:10]} if isinstance(previous, dict) and previous.get("label") in (0, 1) else None) + warm_review_soon() return {} @@ -1183,7 +1301,9 @@ def mark(body): sheet.mark_site_done(ws, int(row), url) elif action == "outcome": # what you heard back: Accepted / Rejected (or back to Applied) from nuauto import setup_sheet - setup_sheet.ensure_dropdown(ws) # (a sheet made before these statuses: its dropdown allows them now) + if (config.SHEET_ID, other) not in _dropdown_set: # (a sheet made before these statuses: its dropdown allows them + setup_sheet.ensure_dropdown(ws) # now; once per start is enough) + _dropdown_set.add((config.SHEET_ID, other)) sheet.set_outcome(ws, int(row), url, body["outcome"]) elif action == "drop": sheet.drop(ws, int(row), url, body["why"]) @@ -1198,6 +1318,9 @@ def mark(body): return {} +_dropdown_set = set() + + def assist_sessions(): """Every assistant session running now: this computer's and (remote_assist) the homelab's.""" from nuauto import assist @@ -1331,9 +1454,8 @@ def other_list(): assistant sessions running now. exists is False until the first job is added (that makes the tab).""" from nuauto import assist from nuauto import intern - try: - rows = sheet.read_rows(worksheet(other=True)) - except NoOtherTab: + rows = other_rows() + if not APP.other_exists: return {"exists": False, "tab": sheet.OTHER_TAB, "ready": [], "applied": [], "rest": [], **assistants()} facts = intern_facts({intern.clean_url(r.url) for r in rows if r.status == "Approved"}) ready, applied, rest = [], [], [] @@ -1372,7 +1494,7 @@ def sheet_get(): out = {} for tab in manage.TABS: try: - rows = sheet.read_rows(worksheet(other=tab == "other")) + rows = read_tab(other=tab == "other") except NoOtherTab: out[tab] = {"exists": False, "rows": []} continue @@ -1381,7 +1503,7 @@ def sheet_get(): return {"tabs": out, "names": manage.TABS, "busy": running("apply", "nuworks_side")} -def jobs_tab(tab="nuworks", order="default"): +def jobs_tab(tab="nuworks", order="default", recheck=False): """One tab of the Apply screen: every row of that sheet tab in sections, what is left to do first, each row with why it can't be moved / removed (None = it can). nuworks: approved (in the order Start goes), agent (the company's own site: the assistant), manual (only you can finish it on NUworks), owed (submitted on NUworks, the company site @@ -1393,7 +1515,10 @@ def jobs_tab(tab="nuworks", order="default"): from nuauto import intern from nuauto import manage other = tab == "other" - rows = other_rows() if other else need_rows() + if recheck: # the screen was just opened: answered from the rows read last, both tabs are read again behind it (a + APP.refresh_soon() # change moves sheet_version, and the screen reloads by itself) + version = APP.sheet_version # (before the rows: a read that lands meanwhile still moves it past this) + rows = other_rows(fresh=False) if other else need_rows() by_number = {r.number: r for r in rows} facts = intern_facts({intern.clean_url(r.url) for r in rows}) if other else None @@ -1414,7 +1539,7 @@ def view(r, **extra): s = {k: [] for k in ("approved", "agent", "manual", "owed", "ready", "applied", "accepted", "rejected", "proposed", "closed", "rest")} a = assistants() - out = {"tab": tab, "name": manage.TABS[tab], "busy": running("apply", "nuworks_side"), **a, + out = {"tab": tab, "name": manage.TABS[tab], "busy": running("apply", "nuworks_side"), **a, "sheet_version": version, "queue": queue_info() if REMOTE else _homelab.get("queue") if a["linked"] and config.REMOTE_ASSIST else None} later = {"Applied": "applied", "Accepted": "accepted", "Rejected": "rejected", "Proposed": "proposed", "Failed": "closed"} if other: @@ -1424,7 +1549,7 @@ def view(r, **extra): s["ready"].append(view(r, target=url, host=urlparse(url).hostname if url else None, why=why)) else: s[later.get(r.status, "rest")].append(view(r)) - out["exists"] = bool(rows) or other_tab_exists() + out["exists"] = bool(rows) or bool(APP.other_exists) else: data = facts_data() run = apply_list(order) @@ -1444,15 +1569,6 @@ def view(r, **extra): return out -def other_tab_exists(): - """Is there an Other jobs tab yet (adding the first job makes it)?""" - try: - worksheet(other=True) - return True - except NoOtherTab: - return False - - def sheet_lookup(url): from nuauto import manage return manage.lookup(url or "") @@ -2244,7 +2360,8 @@ def serve_remote(): q.get("q", "")[:200], q.get("category", "")[:40], "intern" if q.get("source") == "intern" else "nuworks"), "/api/apply": lambda q: apply_list(q.get("order", "default")), - "/api/jobs": lambda q: jobs_tab("other" if q.get("tab") == "other" else "nuworks", q.get("order", "default")), + "/api/jobs": lambda q: jobs_tab("other" if q.get("tab") == "other" else "nuworks", q.get("order", "default"), + q.get("recheck") == "1"), "/api/insights": lambda q: insights_get(q.get("source", "nuworks")), "/api/company": lambda q: company(), "/api/task": lambda q: {"task": APP.task.view(int(q.get("after", 0) or 0)) if APP.task else None}, diff --git a/src/nuauto/gui_static/app.css b/src/nuauto/gui_static/app.css index 8a48e94..3522874 100644 --- a/src/nuauto/gui_static/app.css +++ b/src/nuauto/gui_static/app.css @@ -143,6 +143,7 @@ main.narrow h1 { font-size: 20px; } .btn.small { padding: 5px 10px; font-size: 12.5px; } .btn.big { padding: 11px 18px; font-size: 14.5px; } .btn:disabled { opacity: .5; cursor: not-allowed; } +.btn[aria-busy=true] { cursor: progress; opacity: .7; } /* its click is being done (busyWhile) */ .btn kbd { font: 11px var(--mono); border: 1px solid currentColor; border-radius: 4px; padding: 0 4px; opacity: .6; } input[type=text], input[type=search], input[type=number], input[type=date], input[type=url], input[type=email], select, textarea { font: inherit; color: var(--text); background: var(--surface); border: 1px solid var(--border-strong); @@ -257,6 +258,7 @@ table.grid label.row { flex-wrap: nowrap; white-space: nowrap; gap: 6px; padding .toast { background: var(--text); color: var(--bg); border-radius: 10px; padding: 10px 14px; box-shadow: var(--shadow); display: flex; align-items: center; gap: 12px; max-width: 420px; } .toast.error { background: var(--fail); color: #fff; } +.toast .spinner { border-color: color-mix(in srgb, var(--bg) 35%, transparent); border-top-color: var(--bg); } .toast .btn { background: transparent; color: inherit; border-color: currentColor; padding: 3px 9px; } /* setup wizard */ diff --git a/src/nuauto/gui_static/app.js b/src/nuauto/gui_static/app.js index 54f6217..0cf21f9 100644 --- a/src/nuauto/gui_static/app.js +++ b/src/nuauto/gui_static/app.js @@ -11,6 +11,24 @@ function safeHref(u) { return /^(https?:\/\/|#|\/)/i.test(u) ? u : null; } +// A button (or form) whose handler starts something slower (it returns a promise) shows that at once, and takes no second +// click until it is done +const PENDING = new WeakSet(); +function busyWhile(n, handler) { + return (e) => { + if (PENDING.has(n)) { e.preventDefault(); return undefined; } + const r = handler(e); + if (r && typeof r.then === "function") { + const b = n.tagName === "FORM" ? n.querySelector("[type=submit]") || n : n; + PENDING.add(n); + b.setAttribute("aria-busy", "true"); + const done = () => { PENDING.delete(n); b.removeAttribute("aria-busy"); }; + r.then(done, done); + } + return r; + }; +} + function el(tag, props, ...kids) { const n = document.createElement(tag); for (const [k, v] of Object.entries(props || {})) { @@ -18,7 +36,9 @@ function el(tag, props, ...kids) { if (k === "class") n.className = v; else if (k === "text") n.textContent = v; else if (k === "testid") n.dataset.testid = v; - else if (k.startsWith("on") && typeof v === "function") n.addEventListener(k.slice(2), v); + else if (k.startsWith("on") && typeof v === "function") { + n.addEventListener(k.slice(2), (k === "onclick" && tag === "button") || (k === "onsubmit" && tag === "form") ? busyWhile(n, v) : v); + } else if (PROPS.has(k)) n[k] = v; else if (k === "href" || k === "src") { const h = safeHref(v); if (h) n.setAttribute(k, h); } else n.setAttribute(k, v === true ? "" : String(v)); @@ -96,6 +116,13 @@ function toast(msg, opts = {}) { function fail(e) { toast(e.message || String(e), { error: true }); } +// A change that takes a moment (the sheet): says so at once, until its answer comes +async function working(text, p) { + const t = el("div", { class: "toast", role: "status", testid: "toast-working" }, el("span", { class: "spinner" }), el("span", { text })); + $("#toasts").append(t); + try { return await p; } finally { t.remove(); } +} + function modal(build, { onclose, label = "Dialog", testid = "dialog" } = {}) { const prev = document.activeElement; const box = el("div", { class: "modal", role: "dialog", "aria-modal": "true", "aria-label": label, testid }); @@ -177,7 +204,7 @@ function gone(msg, title) { // ------------------------------------------------------------------ shared state + polling const S = { state: null, checks: [], route: null, screen: null, task: null, taskAfter: 0, taskLog: [], - openQuestion: null, gone: false, lastTaskState: {} }; + openQuestion: null, gone: false, lastTaskState: {}, routes: 0 }; async function pollState() { if (S.gone) return; @@ -185,9 +212,9 @@ async function pollState() { S.state = await api.get("/api/state"); } catch (e) { return; } renderShell(); - // the sheet was read again (a run finished, a row was applied, another window changed it): lists show it at once - const sheetChanged = S.rowsAt !== undefined && S.state.rows_at !== S.rowsAt; - S.rowsAt = S.state.rows_at; + // a read of the sheet found it changed (a run finished, a row was applied, another window changed it): lists show it at once + const sheetChanged = S.sheetVersion !== undefined && S.state.sheet_version !== S.sheetVersion; + S.sheetVersion = S.state.sheet_version; // an assistant started or ended (in its own terminal window): the Apply lists say which rows have one const assistantsChanged = S.assistants !== undefined && S.state.assistants !== S.assistants; S.assistants = S.state.assistants; @@ -604,7 +631,7 @@ function markRow(kind, t, tab) { // tab "other": a row of the sheet's Other job const text = kind === "site" ? `Mark the ${t.company} company-site application as done?` : `Mark ${tab === "other" ? "Other jobs " : ""}row ${t.row} (${t.company}) as Applied today? Only if you submitted it yourself.`; confirmBox(kind === "site" ? "Company site done?" : "You applied yourself?", text, "Yes, mark it", async () => { - try { await api.post("/api/mark", { action: kind, row: t.row, url: t.url, tab }); toast("Sheet updated."); pollState(); if (S.screen && S.screen.reload) S.screen.reload(); } + try { await working("Updating the sheet…", api.post("/api/mark", { action: kind, row: t.row, url: t.url, tab })); toast("Sheet updated."); pollState(); if (S.screen && S.screen.reload) S.screen.reload(); } catch (e) { fail(e); } }); } @@ -614,7 +641,7 @@ function markRow(kind, t, tab) { // tab "other": a row of the sheet's Other job const SENT = ["Applied", "Accepted", "Rejected"]; async function outcomeRow(t, outcome, tab) { try { - await api.post("/api/mark", { action: "outcome", outcome, row: t.row, url: t.url, tab }); + await working("Updating the sheet…", api.post("/api/mark", { action: "outcome", outcome, row: t.row, url: t.url, tab })); toast(outcome === "Applied" ? "Back to Applied." : `Marked ${outcome.toLowerCase()}.`); pollState(); if (S.screen && S.screen.reload) S.screen.reload(); @@ -632,7 +659,7 @@ function dropRow(t, tab) { // tab "other": a row of the sheet's Other jobs tab el("div", { class: "row" }, ...DROP_REASONS.map(([why, label]) => el("button", { class: "btn danger", text: label, testid: `drop-${why}`, onclick: async () => { close(); - try { await api.post("/api/mark", { action: "drop", why, row: t.row, url: t.url, tab }); toast("Dropped."); pollState(); if (S.screen && S.screen.reload) S.screen.reload(); } + try { await working("Dropping it…", api.post("/api/mark", { action: "drop", why, row: t.row, url: t.url, tab })); toast("Dropped."); pollState(); if (S.screen && S.screen.reload) S.screen.reload(); } catch (e) { fail(e); } } })))))); } @@ -1061,18 +1088,21 @@ screens.apply = async (view) => { pane = el("div", { role: "tabpanel", id: "apply-pane", testid: "apply-pane" }, headBox, bar, listBox); fill(view, runBox, tabs, pane); - let reloads = 0; // the newest reload wins: an older one answered last (read before a change) would undo it - async function reload() { - const mine = ++reloads; - const [n, o] = await Promise.allSettled([api.get(`/api/jobs?tab=nuworks&order=${sortOf("nuworks")}`), api.get("/api/jobs?tab=other")]); - if (mine !== reloads) return; - data.nuworks = n.status === "fulfilled" ? n.value : { error: n.reason.message }; - data.other = o.status === "fulfilled" ? o.value : { error: o.reason.message }; - for (const [t, d] of Object.entries(data)) { // each row's key in the homelab's queue (Sort: the queue's order) - if (ok(d)) Object.values(d.sections).flat().forEach((r) => { r.qkey = `${t === "other" ? "other" : "jobs"}-${r.row}`; }); - } - S.queue = data.nuworks.queue || data.other.queue || null; // (the homelab's queue) each company-site row's place in it - draw(); + let reloads = 0, loadedAt = 0; // the newest reload wins: an older one answered last (read before a change) would undo it + async function reload(opening) { // opening: the server answers from the rows read last and reads the sheet again behind it + const mine = ++reloads, again = opening === true ? "&recheck=1" : ""; + loadedAt = Date.now(); + // each tab is drawn as soon as its own answer comes (the other one may still be reading the sheet) + const one = (t, path) => api.get(path).catch((e) => ({ error: e.message })).then((d) => { + if (mine !== reloads) return; + // these rows are as new as that version: the poll does not reload the lists again for it (an open menu stays open) + if (ok(d) && d.sheet_version !== undefined) S.sheetVersion = Math.max(S.sheetVersion || 0, d.sheet_version); + if (ok(d)) Object.values(d.sections).flat().forEach((r) => { r.qkey = `${t === "other" ? "other" : "jobs"}-${r.row}`; }); // its key in the homelab's queue + data[t] = d; + S.queue = (data.nuworks && data.nuworks.queue) || (data.other && data.other.queue) || null; // (the homelab's queue) each company-site row's place in it + draw(); + }); + await Promise.all([one("nuworks", `/api/jobs?tab=nuworks&order=${sortOf("nuworks")}${again}`), one("other", `/api/jobs?tab=other${again}`)]); } function running() { return S.task && S.task.kind === "apply" && S.task.state === "running"; } @@ -1082,7 +1112,7 @@ screens.apply = async (view) => { const d = ok(data.nuworks) ? data.nuworks : ok(data.other) ? data.other : {}; return { d, here: d.sessions || [], there: d.homelab || [] }; } - const todo = (d, keys) => ok(d) ? keys.reduce((n, k) => n + d.sections[k].length, 0) : 0; + const todo = (d, keys) => ok(d) ? keys.reduce((n, k) => n + d.sections[k].length, 0) : d ? 0 : null; // null: not read yet function counts() { const s = sessions(); return { nuworks: todo(data.nuworks, ["approved", "agent", "manual", "owed"]), other: todo(data.other, ["ready"]), @@ -1093,7 +1123,7 @@ screens.apply = async (view) => { const n = counts(), label = { nuworks: "NUworks co-ops", other: otherTabLabel(), assistants: "Assistants" }; fill(tabs, ...TABS.map((k) => el("button", { role: "tab", testid: `tab-${k}`, "aria-selected": String(tab === k), "aria-controls": "apply-pane", onclick: () => { tab = k; sessionStorage.setItem("applyTab", k); drawTabs(); drawPane(); } }, - label[k], el("span", { class: "count", text: String(n[k]) })))); + label[k], el("span", { class: "count", text: n[k] === null ? "…" : String(n[k]) })))); } // Search, kind of work, the order, Add a job: the same for both sheet tabs @@ -1328,12 +1358,13 @@ screens.apply = async (view) => { el("button", { class: "btn small", text: "Follow it here", testid: "btn-follow-homelab", onclick: () => { followHomelabTask(); drawPane(); } })); } - await reload(); + await reload(true); // refresh: the sheet changed (a run finished, a row was applied, something was done elsewhere): read the lists again return { task: () => { drawRun(); if (S.task && S.task.state !== "running") reload(); }, update: () => { // redraw when a run starts or ends here, or one starts or ends on the homelab (its "Follow it here") const ht = S.state && S.state.homelab_link && S.state.homelab_link.task, key = ht ? `${ht.id}:${ht.state}` : ""; if (running() !== drawnRunning || key !== drawnHomelab) { drawnHomelab = key; drawPane(); } + if (S.queue && Date.now() - loadedAt > 30000) reload(); // the homelab's queue moves on by itself (its hours, its waits) }, refresh: reload, reload }; }; @@ -1446,7 +1477,7 @@ function moveRow(names, tab, r, done) { function removeRow(names, tab, r, done) { confirmBox("Remove this job?", `${r.company} · ${r.title} (${names[tab]} row ${r.row}). Its cells are cleared; the other rows keep their numbers.`, "Remove", async () => { - try { await api.post("/api/sheet", { action: "remove", tab, row: r.row, url: r.url }); toast("Removed."); pollState(); done(); } + try { await working("Removing it…", api.post("/api/sheet", { action: "remove", tab, row: r.row, url: r.url })); toast("Removed."); pollState(); done(); } catch (err) { fail(err); } }, { danger: true }); } @@ -2305,9 +2336,12 @@ async function route() { document.querySelectorAll(".side nav a").forEach((a) => a.classList.toggle("active", a.dataset.route === nav)); $("#page-title").textContent = TITLES[key]; document.title = `${TITLES[key]} · NUauto`; - const view = $("#view"); - fill(view, ); - try { S.screen = (await screens[key](view)) || {}; } catch (e) { fail(e); S.screen = {}; } + const view = $("#view"), mine = ++S.routes; + fill(view, el("p", { class: "empty", testid: "screen-loading" }, el("span", { class: "spinner" }), " Loading…")); + let screen; + try { screen = (await screens[key](view)) || {}; } catch (e) { if (mine === S.routes) fail(e); screen = {}; } + if (mine !== S.routes) { if (screen.leave) screen.leave(); return; } // you went on to another screen meanwhile + S.screen = screen; renderTaskbar(); pollState(); // fresh counts for the new screen, not the last poll's view.focus({ preventScroll: true }); diff --git a/src/nuauto/intern.py b/src/nuauto/intern.py index c03236a..7ee60fc 100644 --- a/src/nuauto/intern.py +++ b/src/nuauto/intern.py @@ -708,9 +708,6 @@ def proposed_rows(rows): return {clean_url(r.url): r.number for r in rows or [] if r.url and r.status == "Proposed"} -_TASTE = {} # what the model learned from -> its taste for each pool job (Review asks again on every search) - - def ranked_pool(ratings, other_rows, nuworks_rows=()): """The pool, ordered like the co-ops' (jobs.ranked_pool): by score until the taste model can train, then half score, half taste. The model learns from your co-op and internship decisions together. Returns (pool, trained).""" @@ -723,12 +720,7 @@ def ranked_pool(ratings, other_rows, nuworks_rows=()): i = by_url.get(clean_url(r.url)) if i and r.status in ("Approved", "Applied", "Accepted", "Rejected", "Needs Human", "Failed") and i not in ratings: labels[i] = 1 - key = (json.dumps(labels, sort_keys=True), len(every), tuple(r["id"] for r in pool)) - if key not in _TASTE: - model = jobs.train(every, labels) if pool else None - _TASTE.clear() - _TASTE[key] = {i: float(v) for i, v in model[0]([r["id"] for r in pool]).items()} if model else None - taste = _TASTE[key] + taste = jobs.taste_for(every, labels, [r["id"] for r in pool]) if taste: for r in pool: r["taste"] = taste[r["id"]] diff --git a/src/nuauto/jobs.py b/src/nuauto/jobs.py index 7afd30b..9ad135c 100644 --- a/src/nuauto/jobs.py +++ b/src/nuauto/jobs.py @@ -19,6 +19,7 @@ """ import contextlib import glob +import hashlib import html import json import math @@ -992,16 +993,35 @@ def my_labels(details, ratings, rows): return out +_TASTE = {} # what the model learned from and the jobs it scored -> their taste (Review asks on every load and search) + + +def taste_for(details, labels, ids): + """The taste model's score for each of ids ({id: 0..1}), or None until it can train (train). Trained again only + when the labels, a job's text or ids change: training takes about a second.""" + texts = hashlib.sha256() + for i in sorted(details): + texts.update(f"{i}\0{doc(details[i])}\0".encode()) + key = (texts.hexdigest(), json.dumps(labels, sort_keys=True), tuple(ids)) + if key in _TASTE: + return _TASTE.get(key) + model = train(details, labels) if ids else None + taste = {i: float(v) for i, v in model[0](list(ids)).items()} if model else None + while len(_TASTE) >= 4: # (the co-ops' and the internships' Review, each in both modes) + _TASTE.pop(next(iter(_TASTE)), None) + _TASTE[key] = taste + return taste + + def ranked_pool(details, ratings, rows): - """Pool order: match % until the model can train, then half match %, half your taste.""" + """Pool order: match % until the model can train, then half match %, half your taste. Returns (pool, trained).""" pool = build_pool() - model = train(details, my_labels(details, ratings, rows)) - if model: - taste = model[0]([r["id"] for r in pool]) if pool else {} + taste = taste_for(details, my_labels(details, ratings, rows), [r["id"] for r in pool]) + if taste: for r in pool: - r["taste"] = float(taste[r["id"]]) + r["taste"] = taste[r["id"]] pool.sort(key=pool_sort_key) - return pool, model + return pool, taste is not None def show(r, d): diff --git a/src/nuauto/sheet.py b/src/nuauto/sheet.py index 3f004b7..9c6581c 100644 --- a/src/nuauto/sheet.py +++ b/src/nuauto/sheet.py @@ -7,6 +7,8 @@ import json import os import sys +import threading +import time from dataclasses import dataclass from datetime import date, datetime, timedelta @@ -193,14 +195,13 @@ def client(interactive=True): if not interactive: from google.oauth2.credentials import Credentials config.lock_token() - return gspread.Client(auth=Credentials.from_authorized_user_file(config.TOKEN_PATH), - http_client=gspread.BackOffHTTPClient) + return gspread.Client(auth=Credentials.from_authorized_user_file(config.TOKEN_PATH), http_client=HTTPClient) fresh = not os.path.exists(config.TOKEN_PATH) # gspread will open the browser for a Google login gc = gspread.oauth( scopes=config.WRITE_SCOPES, credentials_filename=config.find_client_json(), authorized_user_filename=config.TOKEN_PATH, - http_client=gspread.BackOffHTTPClient, + http_client=HTTPClient, ) config.lock_token() if fresh: @@ -209,9 +210,78 @@ def client(interactive=True): return gc +class HTTPClient(gspread.BackOffHTTPClient): + """gspread's waiting client; a request Google refuses as unknown (a tab renamed or deleted, the sheet gone) or a login + that can't be renewed also forgets the opened sheet (OPEN), so the next call opens it again.""" + + def request(self, *args, **kwargs): + try: + return super().request(*args, **kwargs) + except gspread.exceptions.APIError as e: + if e.code in (400, 404): + forget() + raise + except RefreshError: + forget() + raise + + +# The opened spreadsheet and its tabs, kept for OPEN_FOR seconds: opening costs two trips to Google before the rows +# (about 0.5 s), every screen of the GUI reads the sheet, and one client keeps its connection open. Rows are always read +# fresh. Opened again when the sheet ID or the Google login (its files) changes, when the login's access token is +# about to expire (renewing it happens only in spreadsheet(), where an expired login is NotLoggedIn), or after a refused +# request (HTTPClient). A tab keeps the one it was opened as: dragging tabs around never makes it another tab. +OPEN_FOR = 300 +OPEN = {} +_open_lock = threading.Lock() + + +def forget(): + """Open the sheet again on the next call (NUauto renamed, added or removed a tab; a request was refused).""" + with _open_lock: + OPEN.clear() + + +def _login_key(): + out = [config.SHEET_ID] + for p in (config.TOKEN_PATH, config.GOOGLE_LOGIN_PATH): + try: + st = os.stat(p) + out.append((st.st_mtime_ns, st.st_size)) + except OSError: + out.append(None) + return tuple(out) + + +def _opened(name): + """The kept spreadsheet (name "book") or tab ("main", "other"), or None.""" + with _open_lock: + if not OPEN or OPEN["key"] != _login_key() or time.monotonic() - OPEN["at"] > OPEN_FOR: + OPEN.clear() + return None + creds = getattr(getattr(OPEN["book"], "client", None), "auth", None) # (demo mode's fake sheet has none) + if creds is not None and not creds.valid: + OPEN.clear() + return None + return OPEN.get(name) + + +def _keep(name, value): + with _open_lock: + if name == "book": + OPEN.clear() + OPEN.update(key=_login_key(), at=time.monotonic(), book=value) + elif OPEN.get("book") is not None: + OPEN[name] = value + return value + + def spreadsheet(interactive=True): + sh = _opened("book") + if sh is not None: + return sh try: - return client(interactive and not config.IS_SERVER).open_by_key(config.SHEET_ID) + sh = client(interactive and not config.IS_SERVER).open_by_key(config.SHEET_ID) except RefreshError: if config.IS_SERVER: raise NotLoggedIn("Google login expired. On the laptop run: nuauto login google") @@ -220,14 +290,18 @@ def spreadsheet(interactive=True): print(f"Google login expired (it lasts {config.GOOGLE_LOGIN_DAYS} days). Opening the browser to log in again...") os.remove(config.TOKEN_PATH) return spreadsheet() + return _keep("book", sh) def open_worksheet(interactive=True): """The main tab (the first one): NUworks jobs only.""" + ws = _opened("main") + if ws is not None: + return ws ws = spreadsheet(interactive).sheet1 if ws.title == OTHER_TAB: raise SheetError(f"The first tab is {OTHER_TAB!r}. Drag your NUworks tab back to the first place.") - return ws + return _keep("main", ws) OTHER_TAB = "Other jobs" # jobs that are not on NUworks (nuauto assist other): same columns, no weekly or total cap @@ -240,9 +314,12 @@ class NoOtherTab(SheetError): def open_other(interactive=True, create=False): """The Other jobs tab. create=True makes it (headers, Status dropdown, styling) if the sheet has none yet; else a missing tab is a SheetError.""" + ws = _opened("other") + if ws is not None: + return ws sh = spreadsheet(interactive) try: - return sh.worksheet(OTHER_TAB) + return _keep("other", sh.worksheet(OTHER_TAB)) except gspread.exceptions.WorksheetNotFound: if not create: raise NoOtherTab(f"No {OTHER_TAB!r} tab yet: add a job to it (NUauto's Sheet screen, or " @@ -250,7 +327,7 @@ def open_other(interactive=True, create=False): from nuauto import setup_sheet ws = sh.add_worksheet(title=OTHER_TAB, rows=setup_sheet.LAST_ROW, cols=len(HEADERS)) setup_sheet.setup(sh, ws, title=OTHER_TAB) - return ws + return _keep("other", ws) SHEETS_API = "https://sheets.googleapis.com/v4/spreadsheets" @@ -285,7 +362,14 @@ def google_login(): def read_rows(ws): - return parse_rows(ws.get_all_values()) + try: + return parse_rows(ws.get_all_values()) + except gspread.exceptions.APIError as e: + # a kept tab renamed or deleted since it was opened: open it again, once + if e.code != 400: + raise + forget() + return parse_rows((open_other if ws.title == OTHER_TAB else open_worksheet)(interactive=False).get_all_values()) def set_status(ws, row_number, status, notes=None, applied_date=None): diff --git a/tests/test_gui.py b/tests/test_gui.py index a1ea71b..a8a6ea4 100644 --- a/tests/test_gui.py +++ b/tests/test_gui.py @@ -150,9 +150,12 @@ def eventually(check, what, seconds=20): rows = json.load(open(os.path.join(STATE, "local", "demo_sheet_DEMO-SHEET.json")))["rows"] assert rows[r["row"] - 1][3] == "Approved" assert json.load(open(os.path.join(STATE, "data", "ratings.json")))[first]["label"] == 1 + # Review's list is kept between loads and made again when what it comes from changes (here: the sheet, a rating) + assert first not in [j["id"] for j in get("/api/review?mode=approve")["jobs"]] post("/api/undo", {"id": first, "row": r["row"], "previous": None}) rows = json.load(open(os.path.join(STATE, "local", "demo_sheet_DEMO-SHEET.json")))["rows"] assert rows[r["row"] - 1][3] == "Proposed" and first not in json.load(open(os.path.join(STATE, "data", "ratings.json"))) + assert first in [j["id"] for j in get("/api/review?mode=approve")["jobs"]] # your Proposed row again post("/api/decide", {"id": "nope", "decision": "approve", "mode": "approve"}, expect=409) post("/api/decide", {"id": rev["jobs"][1]["id"], "decision": "approve", "mode": "rate"}, expect=409) @@ -415,8 +418,22 @@ def eventually(check, what, seconds=20): LAPTOP = "DO ON THE LAPTOP: the homelab's browser was refused" data = json.load(open(other_file)) assert all(x["on_laptop"] is False for x in other["ready"] + other["applied"]) + # the Apply screen is answered from the rows read last; opening it (recheck) reads both tabs again behind it, and + # sheet_version moves only when a read finds a change (then the open screen reloads its lists) + get("/api/jobs?tab=other&recheck=1") + time.sleep(1.5) # (whatever changed before this is read now) + version = get("/api/state")["sheet_version"] + get("/api/jobs?tab=other&recheck=1") + time.sleep(1.5) + assert get("/api/state")["sheet_version"] == version # read again, nothing changed data["rows"][1][4] = f"match 90% | {LAPTOP} (the site showed it a sign-in page with no way in)" json.dump(data, open(other_file, "w")) + get("/api/jobs?tab=other&recheck=1") + for _ in range(50): + if get("/api/state")["sheet_version"] != version: + break + time.sleep(0.1) + assert [x["on_laptop"] for x in get("/api/jobs?tab=other")["sections"]["ready"] if x["row"] == 2] == [True] marked = get("/api/other") assert [(x["row"], x["on_laptop"]) for x in marked["ready"]] == [(2, True)] and marked["applied"][0]["on_laptop"] is False, marked assert marked["ready"][0]["notes"].startswith("match 90% | DO ON THE LAPTOP") @@ -731,7 +748,8 @@ def eventually(check, what, seconds=20): page.reload() page.click("[data-testid=tab-other]") page.wait_for_selector("[data-testid=section-other-ready]") - assert page.locator("[data-testid$=-laptop]").count() == 0 + # (the first answer may be the rows read a moment ago; the sheet is read again behind it and the list follows) + page.wait_for_selector("[data-testid$=-laptop]", state="detached") page.click("[data-testid=tab-nuworks]") manual = get("/api/jobs?tab=nuworks")["sections"]["manual"] # rows only you can finish on NUworks: a section of the same tab assert page.locator("[data-testid=section-nuworks-manual] li").count() == len(manual) diff --git a/tests/test_intern.py b/tests/test_intern.py index d2c2657..f1a9ed5 100644 --- a/tests/test_intern.py +++ b/tests/test_intern.py @@ -567,13 +567,20 @@ def __enter__(self): ratings = {f"x{k}": {"label": k % 2, "date": "x"} for k in range(12)} nuworks = {f"x{k}": {"id": f"x{k}", "title": "Firmware" if k % 2 else "Sales", "skills": [], "function": [], "description": ("firmware C embedded " if k % 2 else "sales marketing ") * 20} for k in range(12)} -real_load = jobs.load_details +real_load, real_train, trains = jobs.load_details, jobs.train, [] jobs.load_details = lambda: nuworks +jobs.train = lambda *a: (trains.append(1), real_train(*a))[1] pool, trained = intern.ranked_pool(ratings, []) -assert trained and all("taste" in r for r in pool) and len(intern._TASTE) == 1 +assert trained and all("taste" in r for r in pool) and trains == [1] pool2, _ = intern.ranked_pool(ratings, []) -assert [r["taste"] for r in pool2] == [r["taste"] for r in pool] and len(intern._TASTE) == 1 -jobs.load_details = real_load +assert [r["taste"] for r in pool2] == [r["taste"] for r in pool] and trains == [1] # not trained again +nuworks["x1"] = {**nuworks["x1"], "description": "firmware sales " * 20} # a job's text changed: trained again +intern.ranked_pool(ratings, []) +assert trains == [1, 1] +ratings["x1"] = {"label": 0, "date": "x"} # a new rating: trained again +intern.ranked_pool(ratings, []) +assert trains == [1, 1, 1] +jobs.load_details, jobs.train = real_load, real_train # summary for the morning message; stats run intern.save("scans.json", [{"time": "2026-10-10T06:00", "listed": 4, "pool": 1}, {"time": "2026-10-08T06:00", "listed": 9, "pool": 9}]) diff --git a/tests/test_sheet.py b/tests/test_sheet.py index b1f3092..7fa6d31 100644 --- a/tests/test_sheet.py +++ b/tests/test_sheet.py @@ -234,6 +234,100 @@ def laptop_ws(status="Needs Human", notes="External application: acme.example -> sheet.client(interactive=True) finally: gspread.Client, gspread.oauth, config.TOKEN_PATH, config.DEMO, config.find_client_json = real -assert made == [("Client", gspread.BackOffHTTPClient), ("oauth", gspread.BackOffHTTPClient)], made +assert made == [("Client", sheet.HTTPClient), ("oauth", sheet.HTTPClient)], made +assert issubclass(sheet.HTTPClient, gspread.BackOffHTTPClient) + +# The opened sheet and its tabs are kept (OPEN_FOR s): a read is one trip to Google, not three. Rows are always fresh. +import time # noqa: E402 +opens = [] + + +class Creds: + valid = True + + +class FakeWs: + def __init__(self, title): + self.title, self.values = title, [HEADERS] + + def get_all_values(self): + return self.values + + +class FakeBook: + def __init__(self): + self.client = type("C", (), {"auth": Creds()})() + self.tabs = [FakeWs("Jobs"), FakeWs(sheet.OTHER_TAB)] + + @property + def sheet1(self): + opens.append("sheet1") + return self.tabs[0] + + def worksheet(self, title): + opens.append("worksheet") + return next(t for t in self.tabs if t.title == title) + + +BOOK = FakeBook() +real_client = sheet.client +sheet.client = lambda interactive=True: type("G", (), {"open_by_key": lambda self, key: (opens.append("book"), BOOK)[1]})() +config.TOKEN_PATH = tok +sheet.forget() +try: + ws = sheet.open_worksheet(interactive=False) + assert sheet.open_worksheet(interactive=False) is ws and sheet.open_other(interactive=False).title == sheet.OTHER_TAB + sheet.open_other(interactive=False) + assert opens == ["book", "sheet1", "worksheet"], opens # opened once each + ws.values = [HEADERS, ["https://x/1", "Co", "Job", "Approved", "", ""]] + assert [r.status for r in sheet.read_rows(sheet.open_worksheet(interactive=False))] == ["Approved"] # fresh rows + BOOK.tabs.reverse() # Other jobs dragged to the first place: the kept main tab is still the NUworks one + assert sheet.open_worksheet(interactive=False).title == "Jobs" + BOOK.tabs.reverse() + opens.clear() + Creds.valid = False # the access token is about to expire: open again (renewing it happens in spreadsheet()) + sheet.open_worksheet(interactive=False) + assert opens == ["book", "sheet1"], opens + Creds.valid = True + opens.clear() + with open(tok, "a") as f: # a new Google login: open again + f.write(" ") + sheet.open_worksheet(interactive=False) + assert opens == ["book", "sheet1"], opens + opens.clear() + sheet.OPEN["at"] = time.monotonic() - sheet.OPEN_FOR - 1 # kept too long + sheet.open_other(interactive=False) + assert opens == ["book", "worksheet"], opens + + # Google refuses a request as unknown (a tab renamed or deleted): the kept sheet is forgotten + class Refused(gspread.BackOffHTTPClient): + def request(self, *a, **k): + raise gspread.exceptions.APIError(type("R", (), {"json": lambda self: {"error": {"code": self.code, "message": "x"}}, + "code": Refused.code, "text": ""})()) + real_request = gspread.BackOffHTTPClient.request + gspread.BackOffHTTPClient.request = Refused.request + try: + for code, forgets in ((429, False), (400, True)): + Refused.code = code + sheet.open_worksheet(interactive=False) + expect(gspread.exceptions.APIError, lambda: sheet.HTTPClient.request(object.__new__(sheet.HTTPClient), "get", "u")) + assert bool(sheet.OPEN) != forgets, code + finally: + gspread.BackOffHTTPClient.request = real_request + + # a kept tab that was renamed: read_rows opens it again, once + old = sheet.open_worksheet(interactive=False) + BOOK.tabs[0] = FakeWs("Jobs") + BOOK.tabs[0].values = [HEADERS, ["https://x/2", "Co", "Job", "Applied", "", ""]] + + def gone(): + sheet.forget() + raise gspread.exceptions.APIError(type("R", (), {"json": lambda self: {"error": {"code": 400, "message": "range"}}, + "text": ""})()) + old.get_all_values = gone + assert [r.status for r in sheet.read_rows(old)] == ["Applied"] +finally: + sheet.client, config.TOKEN_PATH = real_client, real[2] + sheet.forget() print("Submit-safety checks passed.") From 799f99b0aa52a8266ae8cfd38da20940bb0e83b7 Mon Sep 17 00:00:00 2001 From: voidReq <100312344+voidReq@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:09:42 -0400 Subject: [PATCH 2/2] Apply: each answer says the sheet_version its own rows were read at (no second reload that closes an open menu) Co-Authored-By: Claude Opus 5.5 --- src/nuauto/gui.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/nuauto/gui.py b/src/nuauto/gui.py index 095b8c3..aaf320a 100644 --- a/src/nuauto/gui.py +++ b/src/nuauto/gui.py @@ -211,9 +211,10 @@ def __init__(self): self.notified = {} # check id -> (status, time) of the last desktop notification self.first_round_done = False self.rows, self.rows_at, self.rows_error, self.rows_try_at = None, 0, None, 0 + self.rows_version = 0 # sheet_version when these rows were read (the Apply screen's answer says it) self.queue = {} # job id -> pool entry, from the last review queue self.other_urls = None # the Other jobs tab's links (not Proposed), as last read: the internship count - self.other_cache = None # (when, rows) of the Other jobs tab (gui.other_rows) + self.other_cache = None # (when, rows, sheet_version then) of the Other jobs tab (gui.other_rows) self.other_last, self.other_exists = None, None # the tab's rows as last read (kept by other_changed), is there one self.other_try_at, self.other_reading = 0, False self.other_gen = 0 # +1 on each change NUauto makes to the tab: a read begun before it is not kept @@ -353,6 +354,7 @@ def run_group(self, group): if rows != self.rows: self.sheet_version += 1 self.rows, self.rows_at, self.rows_error = rows, now(), None + self.rows_version = self.sheet_version else: self.rows_error = checks[0].detail else: @@ -734,26 +736,31 @@ def todo_list(site, agent, soon): def other_rows(fresh=True): + """other_rows_v's rows.""" + return other_rows_v(fresh)[0] + + +def other_rows_v(fresh=True): """The Other jobs tab's rows ([] before the tab exists); remembers its links for the menu's count. fresh=False: the rows read in the last OTHER_ROWS_FOR seconds will do (each read costs Google quota: about 60 a minute; every - change NUauto makes to the tab forgets them).""" + change NUauto makes to the tab forgets them). other_rows_v: (rows, the sheet_version they are as new as).""" from nuauto import intern cached, gen = APP.other_cache, APP.other_gen if not fresh and cached and now() - cached[0] < OTHER_ROWS_FOR: - return cached[1] + return cached[1], cached[2] try: rows, exists = read_tab(other=True), True except NoOtherTab: rows, exists = [], False with APP.lock: if gen != APP.other_gen: # NUauto changed the tab while this was being read: the next ask reads it again - return rows + return rows, 0 if rows != APP.other_last: APP.sheet_version += 1 APP.other_last, APP.other_exists = rows, exists APP.other_urls = intern.sheet_urls(rows) - APP.other_cache = (now(), rows) - return rows + APP.other_cache = (now(), rows, APP.sheet_version) + return rows, APP.sheet_version def other_changed(): @@ -1517,8 +1524,12 @@ def jobs_tab(tab="nuworks", order="default", recheck=False): other = tab == "other" if recheck: # the screen was just opened: answered from the rows read last, both tabs are read again behind it (a APP.refresh_soon() # change moves sheet_version, and the screen reloads by itself) - version = APP.sheet_version # (before the rows: a read that lands meanwhile still moves it past this) - rows = other_rows(fresh=False) if other else need_rows() + if other: + rows, version = other_rows_v(fresh=False) + else: + need_rows() + with APP.lock: # (the rows and the version they were read at, together) + rows, version = APP.rows, APP.rows_version by_number = {r.number: r for r in rows} facts = intern_facts({intern.clean_url(r.url) for r in rows}) if other else None