Fixes and improvements

2026-07-07 09:29:56 +03:00
parent 784ce673b6
commit ba37eabea9
6 changed files with 238 additions and 15 deletions
@@ -22,6 +22,7 @@ import {
 import { config } from "./config.js";
 import { ensureDatabaseSchema, prisma } from "./db.js";
 import { generateExpectedScreenshot, judgeSubmission } from "./judge.js";
+import { queuedJobCount, queueSequential } from "./sequentialQueue.js";
 import type { CheckDefinition } from "./types.js";

 const upload = multer({
@@ -37,6 +38,10 @@ const checkSchema = z.object({
      "attribute-equals",
      "css-property",
      "element-count",
+      "interaction-text-contains",
+      "interaction-attribute-equals",
+      "interaction-css-property",
+      "interaction-element-count",
      "visual-match"
    ],
    { errorMap: () => ({ message: "Choose a supported check type." }) }
@@ -45,6 +50,9 @@ const checkSchema = z.object({
  value: z.string().optional(),
  attribute: z.string().optional(),
  property: z.string().optional(),
+  actionSelector: z.string().optional(),
+  inputSelector: z.string().optional(),
+  inputValue: z.string().optional(),
  count: z.coerce.number({ invalid_type_error: "Enter a valid element count." }).int().nonnegative().optional(),
  threshold: z.coerce.number({ invalid_type_error: "Enter a valid visual threshold." }).min(0).max(1).optional(),
  weight: z.coerce.number({ invalid_type_error: "Enter a valid check weight." }).positive("Check weight must be greater than 0.").default(1),
@@ -55,7 +63,8 @@ function validateChecks(checks: z.infer<typeof checkSchema>[], context: z.Refine
  checks.forEach((check, index) => {
    const label = `Check ${index + 1}`;
    const needsSelector = check.type !== "visual-match";
-    const needsValue = ["text-contains", "attribute-equals", "css-property"].includes(check.type);
+    const needsValue = ["text-contains", "attribute-equals", "css-property", "interaction-text-contains", "interaction-attribute-equals", "interaction-css-property"].includes(check.type);
+    const isInteraction = check.type.startsWith("interaction-");

    if (needsSelector && !check.selector?.trim()) {
      context.addIssue({
@@ -73,7 +82,15 @@ function validateChecks(checks: z.infer<typeof checkSchema>[], context: z.Refine
      });
    }

-    if (check.type === "attribute-equals" && !check.attribute?.trim()) {
+    if (isInteraction && !check.actionSelector?.trim()) {
+      context.addIssue({
+        code: z.ZodIssueCode.custom,
+        path: ["checks", index, "actionSelector"],
+        message: `${label}: enter the button/action selector to click.`
+      });
+    }
+
+    if (["attribute-equals", "interaction-attribute-equals"].includes(check.type) && !check.attribute?.trim()) {
      context.addIssue({
        code: z.ZodIssueCode.custom,
        path: ["checks", index, "attribute"],
@@ -81,7 +98,7 @@ function validateChecks(checks: z.infer<typeof checkSchema>[], context: z.Refine
      });
    }

-    if (check.type === "css-property" && !check.property?.trim()) {
+    if (["css-property", "interaction-css-property"].includes(check.type) && !check.property?.trim()) {
      context.addIssue({
        code: z.ZodIssueCode.custom,
        path: ["checks", index, "property"],
@@ -89,7 +106,7 @@ function validateChecks(checks: z.infer<typeof checkSchema>[], context: z.Refine
      });
    }

-    if (check.type === "element-count" && check.count === undefined) {
+    if (["element-count", "interaction-element-count"].includes(check.type) && check.count === undefined) {
      context.addIssue({
        code: z.ZodIssueCode.custom,
        path: ["checks", index, "count"],
@@ -367,11 +384,15 @@ async function maybeGenerateExpected(task: {
    return null;
  }

-  return generateExpectedScreenshot(task.id, {
+  return queueSequential(() => generateExpectedScreenshot(task.id, {
    html: task.referenceHtml,
    css: task.referenceCss,
    js: task.referenceJs
-  });
+  }));
+}
+
+function queueJudgeSubmission(options: Parameters<typeof judgeSubmission>[0]) {
+  return queueSequential(() => judgeSubmission(options));
 }

 async function seedSampleData() {
@@ -451,7 +472,7 @@ export async function createApp() {
  app.use("/generated", express.static(config.generatedDir));

  app.get("/api/health", (_req, res) => {
-    res.json({ ok: true });
+    res.json({ ok: true, queuedJobs: queuedJobCount() });
  });

  app.post("/api/auth/admin", (req, res) => {
@@ -569,7 +590,7 @@ export async function createApp() {
    }

    const studentName = normalizeStudentName(parsed.data.studentName);
-    const result = await judgeSubmission({
+    const result = await queueJudgeSubmission({
      taskId: task.id,
      checksJson: task.checksJson,
      expectedScreenshotPath: task.expectedScreenshotPath,
@@ -845,7 +866,7 @@ export async function createApp() {
      return;
    }

-    const result = await judgeSubmission({
+    const result = await queueJudgeSubmission({
      taskId: submission.taskId,
      checksJson: submission.task.checksJson,
      expectedScreenshotPath: submission.task.expectedScreenshotPath,
@@ -902,10 +923,16 @@ export async function createApp() {
    res.json({ url, markdown: `![Downloaded image](${url})` });
  });

-  if (config.nodeEnv === "production") {
+  const frontendIndexPath = path.join(config.frontendDistDir, "index.html");
+  const hasBuiltFrontend = await fs
+    .access(frontendIndexPath)
+    .then(() => true)
+    .catch(() => false);
+
+  if (config.nodeEnv === "production" || hasBuiltFrontend) {
    app.use(express.static(config.frontendDistDir));
    app.get("*", (_req, res) => {
-      res.sendFile(path.join(config.frontendDistDir, "index.html"));
+      res.sendFile(frontendIndexPath);
    });
  }

@@ -55,6 +55,20 @@ function checkWeight(check: CheckDefinition) {
  return typeof check.weight === "number" && Number.isFinite(check.weight) && check.weight > 0 ? check.weight : 1;
 }

+function isInteractionCheck(check: CheckDefinition) {
+  return check.type.startsWith("interaction-");
+}
+
+async function runInteraction(page: Awaited<ReturnType<typeof renderScreenshot>>["page"], check: CheckDefinition) {
+  if (check.inputSelector) {
+    await page.locator(check.inputSelector).first().fill(check.inputValue ?? "");
+  }
+  if (check.actionSelector) {
+    await page.locator(check.actionSelector).first().click();
+    await page.waitForTimeout(50);
+  }
+}
+
 export function summarizeWeightedResults(results: CheckResult[], passingScore = 0.8) {
  const totalWeight = results.reduce((sum, result) => sum + (result.weight ?? 1), 0);
  const passedWeight = results.reduce((sum, result) => sum + (result.passed ? result.weight ?? 1 : 0), 0);
@@ -222,6 +236,97 @@ export async function judgeSubmission(options: {
        continue;
      }

+      if (isInteractionCheck(check)) {
+        if (!check.selector || !check.actionSelector) {
+          results.push({
+            type: check.type,
+            message: check.message,
+            passed: false,
+            weight: checkWeight(check),
+            details: "This interaction check is missing a result selector or action selector."
+          });
+          continue;
+        }
+
+        const interactionPage = await page.context().newPage();
+        interactionPage.setDefaultTimeout(1500);
+        interactionPage.on("dialog", (dialog) => dialog.dismiss().catch(() => undefined));
+
+        try {
+          await interactionPage.setContent(buildHtmlDocument(options.bundle), {
+            waitUntil: "domcontentloaded",
+            timeout: 2000
+          });
+          await interactionPage.waitForTimeout(50);
+          await runInteraction(interactionPage, check);
+        } catch (error) {
+          await interactionPage.close().catch(() => undefined);
+          results.push({
+            type: check.type,
+            message: check.message,
+            passed: false,
+            weight: checkWeight(check),
+            details: error instanceof Error ? error.message : "Could not perform the interaction."
+          });
+          continue;
+        }
+
+        const locator = interactionPage.locator(check.selector);
+        const count = await locator.count();
+
+        if (check.type === "interaction-text-contains") {
+          const text = count > 0 ? await locator.first().textContent() : "";
+          const expected = check.value ?? "";
+          results.push({
+            type: check.type,
+            message: check.message,
+            passed: Boolean(text?.toLocaleLowerCase("en-US").includes(expected.toLocaleLowerCase("en-US"))),
+            weight: checkWeight(check),
+            details: `After clicking ${check.actionSelector}, expected text containing "${expected}".`
+          });
+        }
+
+        if (check.type === "interaction-element-count") {
+          const expectedCount = check.count ?? Number(check.value ?? 0);
+          results.push({
+            type: check.type,
+            message: check.message,
+            passed: count === expectedCount,
+            weight: checkWeight(check),
+            details: `After clicking ${check.actionSelector}, found ${count}, expected ${expectedCount}.`
+          });
+        }
+
+        if (check.type === "interaction-attribute-equals") {
+          const actual = count > 0 ? await locator.first().getAttribute(check.attribute ?? "") : null;
+          const expected = check.value ?? "";
+          results.push({
+            type: check.type,
+            message: check.message,
+            passed: actual === expected,
+            weight: checkWeight(check),
+            details: `After clicking ${check.actionSelector}, expected ${check.attribute}="${expected}", found ${actual ?? "nothing"}.`
+          });
+        }
+
+        if (check.type === "interaction-css-property") {
+          const actual = count > 0
+            ? await locator.first().evaluate((el, property) => getComputedStyle(el).getPropertyValue(property), check.property ?? "")
+            : "";
+          const expected = check.value ?? "";
+          results.push({
+            type: check.type,
+            message: check.message,
+            passed: actual.trim() === expected.trim(),
+            weight: checkWeight(check),
+            details: `After clicking ${check.actionSelector}, expected ${check.property}: ${expected}, found ${actual || "nothing"}.`
+          });
+        }
+
+        await interactionPage.close().catch(() => undefined);
+        continue;
+      }
+
      if (!check.selector) {
        results.push({
          type: check.type,
@@ -0,0 +1,27 @@
+type Job<T> = () => Promise<T>;
+
+let tail = Promise.resolve();
+let pendingJobs = 0;
+
+export function queueSequential<T>(job: Job<T>): Promise<T> {
+  pendingJobs += 1;
+
+  const run = tail.then(async () => {
+    try {
+      return await job();
+    } finally {
+      pendingJobs -= 1;
+    }
+  });
+
+  tail = run.then(
+    () => undefined,
+    () => undefined
+  );
+
+  return run;
+}
+
+export function queuedJobCount() {
+  return pendingJobs;
+}
@@ -4,6 +4,10 @@ export type CheckType =
  | "attribute-equals"
  | "css-property"
  | "element-count"
+  | "interaction-text-contains"
+  | "interaction-attribute-equals"
+  | "interaction-css-property"
+  | "interaction-element-count"
  | "visual-match";

 export type CheckDefinition = {
@@ -13,6 +17,9 @@ export type CheckDefinition = {
  value?: string;
  attribute?: string;
  property?: string;
+  actionSelector?: string;
+  inputSelector?: string;
+  inputValue?: string;
  count?: number;
  threshold?: number;
  weight?: number;