From 8d901831b20492016132cd94e77b5991a7f66a9c Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Fri, 25 Sep 2026 18:11:14 +0200 Subject: [PATCH] push all scripts --- .idea/copilotDiffState.xml | 18 +++ docs/smarttime-9.md | 90 +++++++++++ scripts/smarttime_cli.py | 39 +++++ scripts/smarttime_probe.py | 82 ++++++++++ tests/fixtures/hung_smarttime_browser.py | 12 ++ tests/test_smarttime_ui.py | 191 +++++++++++++++++++++++ 6 files changed, 432 insertions(+) create mode 100644 .idea/copilotDiffState.xml create mode 100644 docs/smarttime-9.md create mode 100644 scripts/smarttime_cli.py create mode 100644 scripts/smarttime_probe.py create mode 100644 tests/fixtures/hung_smarttime_browser.py create mode 100644 tests/test_smarttime_ui.py diff --git a/.idea/copilotDiffState.xml b/.idea/copilotDiffState.xml new file mode 100644 index 0000000..b2ded4e --- /dev/null +++ b/.idea/copilotDiffState.xml @@ -0,0 +1,18 @@ + + + + + + \ No newline at end of file diff --git a/docs/smarttime-9.md b/docs/smarttime-9.md new file mode 100644 index 0000000..5908d78 --- /dev/null +++ b/docs/smarttime-9.md @@ -0,0 +1,90 @@ +# SmartTime 9 UI integration + +Observed read-only against the internal instance on 2026-09-25 using a separate +headless Playwright browser. No stamps, corrections or approvals were submitted. +The dashboard and September month row both showed **09:29**, not exactly 09:30. + +## Changes from 8.6 + +- Login redirects to `/login`; the user input is now `input[name="username"]`, + not `useraccount`. The password input remains `input[name="password"]`. +- The application is Vaadin Flow with web components and shadow DOM, not the old + GWT dashboard. Setting `location.hash = 'dashboard_tab'` is obsolete. +- Sidebar items are `vaadin-side-nav-item[path="..."]`. Dashboard has an empty + path, month view has `MonthlyOV`; the selected item has a `current` attribute. +- The personal card still has a `Letzte Buchung` label followed by an HH:MM label. + It is inside `.shadow-xs`. Its `vaadin-icon[icon="far:building"]` uses the + status color. The phone icon is not the presence indicator. +- There is no `Aktueller Status` value. The status legend contains all four labels + regardless of actual presence. Matching the first `Anwesend` in body text is + unsafe. Match the personal building icon's computed fill to each legend swatch's + computed background color; unknown or ambiguous colors fail closed. + +## Loading and recovery + +Login, navigation and monthly selections finish asynchronously. Wait for the +expected DOM and the Vaadin Flow clients' `isActive()` queues to become idle. +Do not use `networkidle`: the app maintains background communication. Menu options +remain in the DOM after closing; locate visible top-level menu buttons separately. + +`smarttime_browser.py` launches one isolated worker per read. A whole-job watchdog +also covers cases in which a Playwright DOM operation or browser shutdown hangs. +On Windows it terminates only that job's process tree (`taskkill /PID /T /F`) +before retrying with a new browser. There is no browser profile lock or poisoned +thread queue to reuse after a failure. The old profile is left untouched on disk. +The subprocess boundary also resolves Streamlit's Windows SelectorEventLoop issue. + +UI waits are bounded (normally 40 seconds); the hard job budgets are 75 seconds for +status and 180 seconds for export, with at most three read attempts. Exhaustion is +an unknown/error result, never proof that a booking failed. + +## Monthly extraction + +1. Click `vaadin-side-nav-item[path="MonthlyOV"]` and wait for `current`. +2. Read the visible month/year `vaadin-menu-bar-button` labels. Select the year and + month by their exact menu-item names; wait for the new labels and idle server queue. +3. Read `vaadin-grid.Customers-grid`. Its shadow root contains `#header`, `#items` + and the scrolling `#table`. Each cell's `` resolves to light-DOM content + via `assignedElements()`. This preserves empty columns and multiple bookings. +4. Use the grid's `scrollToIndex()` to render successive portions of the month. + The grid's `size` must equal the calendar's number of days. Collect by day, + compare overlapping rows, and require every date before writing the export. + +Observed columns, in order: `Datum`, an unlabeled notes/icon column, +`Kommen / Gehen`, `Soll`, `Ist`, `TSaldo`, `Saldo`, `Pause`, `Abwesenheit`. +The unlabeled column is retained in `raw_cells`. Booking strings are kept in +display order; absence credits are not turned into fictitious bookings. + +## CLI and diagnostics + +`scripts/smarttime_cli.py` uses exactly the same reader/recovery code as the app. +`--month YYYY-MM` exports JSON; `--artifacts DIRECTORY` saves a screenshot and +sanitized HTML for the dashboard and multiple month scroll positions. The HTML +includes declarative shadow-root snapshots; ordinary `page.content()` omits them. +Input value attributes are stripped, but exported time data remains personal. + +`scripts/smarttime_probe.py` is the separate interactive exploration tool. It uses +a fresh headless browser, accepts JSON commands on stdin, and saves local snapshots +under `.local/smarttime/`. Example commands: + +```json +{"action":"snapshot"} +{"action":"login","user":"input[name=username]","password":"input[name=password]"} +{"action":"selector","selector":"a[href=MonthlyOV]"} +{"action":"snapshot"} +{"action":"quit"} +``` + +The probe loads credentials from settings without printing them. Its generic +selectors are intended for developer inspection; use only navigation/read controls. +The supported CLI exposes only read operations. + +## Validation + +Live dashboard reads, the app's verification API, the Streamlit **Jetzt prüfen** +button, and full August/September 2026 and September 2025 exports were exercised. +The Streamlit check used the live backend with the auto-stamp scheduler mocked out. +Offline tests cover the real DOM-reading JavaScript, +status colors, delayed rendering, unknown/invalid values, slot extraction, full +calendar validation, and retry guards. A separate test launches real Chromium +with an intercepted navigation that never responds, then verifies watchdog cleanup. diff --git a/scripts/smarttime_cli.py b/scripts/smarttime_cli.py new file mode 100644 index 0000000..7b83ace --- /dev/null +++ b/scripts/smarttime_cli.py @@ -0,0 +1,39 @@ +"""Read SmartTime status or export a complete month, without starting Streamlit.""" +import argparse +import json +import sys +from datetime import datetime +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "stempelbot")) + +from smarttime_browser import run_browser_job + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--month", type=lambda s: datetime.strptime(s, "%Y-%m"), help="Export YYYY-MM; omit for current dashboard status") + parser.add_argument("--output", type=Path, help="Write JSON to a new file (otherwise stdout)") + parser.add_argument("--artifacts", type=Path, help="Save local screenshots and sanitized HTML snapshots") + args = parser.parse_args() + if args.output and args.output.exists(): + parser.error("Output already exists; choose a new file") + result = run_browser_job( + "month" if args.month else "status", + year=args.month.year if args.month else None, + month=args.month.month if args.month else None, + artifacts=args.artifacts, + ) + payload = json.dumps(result, ensure_ascii=False, indent=2) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x", encoding="utf-8") as stream: + stream.write(payload + "\n") + print(f"Saved {args.output}") + else: + print(payload) + + +if __name__ == "__main__": + main() diff --git a/scripts/smarttime_probe.py b/scripts/smarttime_probe.py new file mode 100644 index 0000000..f8e6479 --- /dev/null +++ b/scripts/smarttime_probe.py @@ -0,0 +1,82 @@ +"""Interactive read-only SmartTime UI probe (separate from the app). + +Reads JSON commands from stdin: snapshot, login, click (text), goto (url), quit. +Artifacts stay local; no credential or cookie values are printed. +""" +import json +import sys +from datetime import datetime +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "stempelbot")) + +from playwright.sync_api import sync_playwright +from settings import settings + + +def main(): + output = ROOT / ".local" / "smarttime" / datetime.now().strftime("%Y%m%d-%H%M%S") + output.mkdir(parents=True, exist_ok=True) + with sync_playwright() as pw: + browser = pw.chromium.launch(headless=True) + context = browser.new_context(ignore_https_errors=True, locale="de-DE", viewport={"width": 1440, "height": 1100}) + page = context.new_page() + page.set_default_timeout(8000) + page.goto(settings.SMART_TIME_URL, wait_until="domcontentloaded", timeout=20000) + print(f"ARTIFACTS {output}", flush=True) + count = 0 + while True: + try: + command = json.loads(input("COMMAND> ")) + action = command["action"] + if action == "quit": + break + if action == "evaluate": + print(json.dumps(page.evaluate(command["js"]), ensure_ascii=True), flush=True) + continue + if action == "run": + import runpy + runpy.run_path(command["file"])["run"](page, output) + continue + if action == "login": + page.locator(command["user"]).fill(settings.USERNAME) + page.locator(command["password"]).fill(settings.SMART_TIME_PASSWORD) + page.locator(command["password"]).press("Enter") + elif action == "click": + page.get_by_text(command["text"], exact=True).click() + elif action == "selector": + page.locator(command["selector"]).click() + elif action == "fill": + page.locator(command["selector"]).fill(command["value"]) + page.locator(command["selector"]).press("Enter") + elif action == "goto": + page.goto(command["url"], wait_until="domcontentloaded", timeout=20000) + elif action == "inspect": + print(page.locator(command["selector"]).evaluate_all("els => els.map(e => e.outerHTML)"), flush=True) + count += 1 + name = f"{count:02d}-{action}" + page.screenshot(path=str(output / f"{name}.png"), full_page=True, timeout=10000) + # Clone DOM and remove input values before persisting snapshots. + html = page.evaluate("""() => { const root = document.documentElement.cloneNode(true); + root.querySelectorAll('input').forEach(e => e.removeAttribute('value')); + return '' + root.outerHTML; }""") + (output / f"{name}.html").write_text(html, encoding="utf-8") + print("URL", page.url, flush=True) + print(page.locator("body").inner_text(timeout=5000), flush=True) + if action == "inspect" or "login" in page.url: + print("CONTROLS", page.locator("input,button,a,select,[role=tab]").evaluate_all("els => els.map(e => ({tag:e.tagName, name:e.getAttribute('name'), type:e.getAttribute('type'), id:e.id, text:e.innerText, title:e.title, href:e.getAttribute('href')}))"), flush=True) + except EOFError: + break + except Exception as exc: + message = str(exc) + for secret in (settings.SMART_TIME_PASSWORD, settings.PASSWORD): + if secret: + message = message.replace(secret, "***") + print(type(exc).__name__, message, flush=True) + context.close() + browser.close() + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/hung_smarttime_browser.py b/tests/fixtures/hung_smarttime_browser.py new file mode 100644 index 0000000..acab5a4 --- /dev/null +++ b/tests/fixtures/hung_smarttime_browser.py @@ -0,0 +1,12 @@ +"""Offline fixture: stall a Chromium navigation forever until the parent kills it.""" +import json +import sys +from playwright.sync_api import sync_playwright + +json.load(sys.stdin) +with sync_playwright() as pw: + browser = pw.chromium.launch(headless=True) + page = browser.new_page() + # Intercept locally: there is no request to the network or to SmartTime. + page.route("**/*", lambda route: None) + page.goto("https://smarttime-hang.invalid/", timeout=0) diff --git a/tests/test_smarttime_ui.py b/tests/test_smarttime_ui.py new file mode 100644 index 0000000..6236026 --- /dev/null +++ b/tests/test_smarttime_ui.py @@ -0,0 +1,191 @@ +import calendar +import json +import subprocess +import sys +import time +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "stempelbot")) + +from playwright.sync_api import sync_playwright +import smarttime_browser as browser_jobs +import smarttime_ui as ui + + +# Minimal, anonymized markup matching the observed SmartTime 9 dashboard. +DASHBOARD = """ +
+ + + +
+
 +
 +
 +
 +""" +HEADERS = ["Datum", "", "Kommen / Gehen", "Soll", "Ist", "TSaldo", "Saldo", "Pause", "Abwesenheit"] + + +class DashboardDOMTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.pw = sync_playwright().start() + cls.browser = cls.pw.chromium.launch(headless=True) + + @classmethod + def tearDownClass(cls): + cls.browser.close() + cls.pw.stop() + + def setUp(self): + self.page = self.browser.new_page() + self.addCleanup(self.page.close) + + def test_status_comes_from_personal_card_not_legend_or_phone(self): + for color, status in (("green", "Anwesend"), ("red", "Abwesend"), + ("rgb(31, 161, 220)", "Außer Haus"), ("orange", "Entschuldigt")): + with self.subTest(status=status): + self.page.set_content(DASHBOARD.replace("COLOR", color)) + result = ui.read_dashboard(self.page) + self.assertEqual(result["last_stamp"], "09:29") + self.assertEqual(result["status_text"], status) + + def test_unknown_status_is_not_assumed_present(self): + self.page.set_content(DASHBOARD.replace("COLOR", "purple")) + with self.assertRaisesRegex(ui.SmartTimeReadError, "complete personal dashboard"): + ui.read_dashboard(self.page, timeout=0) + + def test_waits_for_delayed_dashboard_values(self): + self.page.set_content(DASHBOARD.replace("COLOR", "green").replace("09:29", "")) + self.page.evaluate("""() => setTimeout(() => { + document.querySelector('.shadow-xs vaadin-vertical-layout label:last-child').textContent='09:29'; + }, 100)""") + self.assertEqual(ui.read_dashboard(self.page)["last_stamp"], "09:29") + + def test_invalid_time_is_not_accepted(self): + self.page.set_content(DASHBOARD.replace("COLOR", "green").replace("09:29", "29:90")) + with self.assertRaises(ui.SmartTimeReadError): + ui.read_dashboard(self.page, timeout=0) + + def test_empty_grid_can_finish_rendering(self): + self.page.set_content('') + self.page.evaluate("() => document.querySelector('vaadin-grid').attachShadow({mode:'open'})") + self.assertIsNone(self.page.evaluate(ui.GRID_JS)) + + def test_virtual_grid_reads_slots_and_preserves_empty_columns(self): + self.page.set_content('') + values = [HEADERS, ["25", "", "09:29", "08:00", "00:00", "-08:00", "22:17", "00:00", ""]] + self.page.evaluate("""values => { + const g = document.querySelector('vaadin-grid'); g.size = 30; + const root = g.attachShadow({mode:'open'}); + const table = document.createElement('div'); table.id='table'; root.append(table); + values.forEach((cells, i) => { + const section = document.createElement('div'); section.id = i ? 'items' : 'header'; + const row = document.createElement('tr'); section.append(row); table.append(section); + cells.forEach((text, j) => { + const slot = document.createElement('slot'); slot.name=`cell-${i}-${j}`; row.append(slot); + const content = document.createElement('span'); content.slot=slot.name; content.innerText=text; g.append(content); + }); + }); + }""", values) + result = self.page.evaluate(ui.GRID_JS) + self.assertEqual(result["headers"], HEADERS) + self.assertEqual(result["rows"], [values[1]]) + + +class MonthExportTests(unittest.TestCase): + def rows(self, year=2026, month=9): + return [[f"{d:02d}", "", "", "08:00", "00:00", "-08:00", "", "00:00", ""] + for d in range(1, calendar.monthrange(year, month)[1] + 1)] + + def test_preserves_all_stamps_totals_absence_and_open_pair(self): + rows = self.rows() + rows[0][2] = "09:00\n12:00\n12:30\n17:45" + rows[0][4] = "08:15" + rows[1][8] = "Urlaub" + rows[24][2] = "09:29" + result = ui.normalize_month(2026, 9, HEADERS, list(reversed(rows))) + self.assertEqual(len(result["days"]), 30) + self.assertEqual(result["days"][0]["stamps"], ["09:00", "12:00", "12:30", "17:45"]) + self.assertEqual(result["days"][0]["actual"], "08:15") + self.assertEqual(result["days"][1]["absence"], "Urlaub") + self.assertTrue(result["days"][24]["incomplete_pair"]) + self.assertEqual(result["days"][24]["raw_cells"], rows[24]) + + def test_missing_duplicate_or_changed_columns_fail_closed(self): + rows = self.rows() + for headers, values in ((HEADERS, rows[:-1]), (HEADERS, rows[:-1] + [rows[0]]), + (HEADERS[::-1], rows)): + with self.subTest(headers=headers, count=len(values)): + with self.assertRaises(ui.SmartTimeReadError): + ui.normalize_month(2026, 9, headers, values) + + def test_leap_month_has_all_29_dates(self): + self.assertEqual(len(ui.normalize_month(2024, 2, HEADERS, self.rows(2024, 2))["days"]), 29) + + +class BrowserRecoveryTests(unittest.TestCase): + def process(self, response=None, *, error=None): + process = Mock() + process.returncode = 0 + process.poll.return_value = 0 + if error: + process.communicate.side_effect = error + else: + process.communicate.return_value = (json.dumps(response), "") + return process + + def test_hang_starts_fresh_browser_and_recovers(self): + hung = self.process(error=subprocess.TimeoutExpired("browser", 75)) + good = self.process({"result": {"last_stamp": "09:29"}}) + with (patch.object(browser_jobs.subprocess, "Popen", side_effect=[hung, good]) as spawn, + patch.object(browser_jobs, "stop_process_tree") as cleanup, + patch.object(browser_jobs.time, "sleep")): + self.assertEqual(browser_jobs.run_browser_job(), {"last_stamp": "09:29"}) + self.assertEqual(spawn.call_count, 2) + self.assertEqual(cleanup.call_count, 2) + hung.stdin.close.assert_called_once() + + def test_permanent_hang_is_bounded_and_next_job_can_recover(self): + processes = [self.process(error=subprocess.TimeoutExpired("browser", 75)) for _ in range(3)] + with (patch.object(browser_jobs.subprocess, "Popen", side_effect=processes) as spawn, + patch.object(browser_jobs, "stop_process_tree"), patch.object(browser_jobs.time, "sleep")): + with self.assertRaisesRegex(RuntimeError, "after 3 browser reads"): + browser_jobs.run_browser_job() + self.assertEqual(spawn.call_count, 3) + + def test_invalid_credentials_are_not_retried(self): + rejected = self.process({"error": "SmartTime rejected the login", "login_error": True}) + rejected.returncode = 1 + with (patch.object(browser_jobs.subprocess, "Popen", return_value=rejected) as spawn, + patch.object(browser_jobs, "stop_process_tree")): + with self.assertRaises(PermissionError): + browser_jobs.run_browser_job() + spawn.assert_called_once() + + def test_real_hung_playwright_process_is_terminated(self): + fixture = Path(__file__).parent / "fixtures" / "hung_smarttime_browser.py" + processes = [] + original_spawn = subprocess.Popen + + def spawn(*args, **kwargs): + process = original_spawn(*args, **kwargs) + processes.append(process) + return process + + started = time.monotonic() + with (patch.object(browser_jobs, "__file__", str(fixture)), + patch.object(browser_jobs, "STATUS_TIMEOUT_SEC", 3), + patch.object(browser_jobs, "BROWSER_ATTEMPTS", 1), + patch.object(browser_jobs.subprocess, "Popen", side_effect=spawn)): + with self.assertRaisesRegex(RuntimeError, "did not respond within 3s"): + browser_jobs.run_browser_job() + self.assertIsNotNone(processes[0].poll()) + self.assertLess(time.monotonic() - started, 15) + + +if __name__ == "__main__": + unittest.main()