Initial commit
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"name": "@frontend-judge/backend",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/server.ts",
|
||||
"build": "prisma generate && tsc",
|
||||
"start": "node dist/server.js",
|
||||
"test": "vitest run",
|
||||
"prisma:generate": "prisma generate",
|
||||
"prisma:migrate": "prisma migrate dev --name init"
|
||||
},
|
||||
"dependencies": {
|
||||
"@prisma/client": "^6.1.0",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"dotenv": "^16.6.1",
|
||||
"express": "^4.21.2",
|
||||
"helmet": "^8.0.0",
|
||||
"multer": "^2.2.0",
|
||||
"pixelmatch": "^6.0.0",
|
||||
"playwright": "^1.49.1",
|
||||
"pngjs": "^7.0.0",
|
||||
"zod": "^3.24.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/cookie-parser": "^1.4.8",
|
||||
"@types/express": "^5.0.0",
|
||||
"@types/multer": "^1.4.12",
|
||||
"@types/node": "^22.10.2",
|
||||
"@types/pixelmatch": "^5.2.6",
|
||||
"@types/pngjs": "^6.0.5",
|
||||
"prisma": "^6.1.0",
|
||||
"supertest": "^7.0.0",
|
||||
"tsx": "^4.19.2",
|
||||
"typescript": "^5.7.2",
|
||||
"vitest": "^2.1.8"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
generator client {
|
||||
provider = "prisma-client-js"
|
||||
}
|
||||
|
||||
datasource db {
|
||||
provider = "sqlite"
|
||||
url = env("DATABASE_URL")
|
||||
}
|
||||
|
||||
model DayGroup {
|
||||
id String @id @default(cuid())
|
||||
title String
|
||||
order Int @default(0)
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
tasks Task[]
|
||||
}
|
||||
|
||||
model Task {
|
||||
id String @id @default(cuid())
|
||||
title String
|
||||
descriptionMarkdown String
|
||||
groupId String
|
||||
order Int @default(0)
|
||||
visible Boolean @default(true)
|
||||
starterHtml String @default("")
|
||||
starterCss String @default("")
|
||||
starterJs String @default("")
|
||||
referenceHtml String @default("")
|
||||
referenceCss String @default("")
|
||||
referenceJs String @default("")
|
||||
expectedScreenshotPath String?
|
||||
visualThreshold Float @default(0.85)
|
||||
passingScore Float @default(0.8)
|
||||
checksJson String @default("[]")
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
group DayGroup @relation(fields: [groupId], references: [id], onDelete: Cascade)
|
||||
submissions Submission[]
|
||||
}
|
||||
|
||||
model Submission {
|
||||
id String @id @default(cuid())
|
||||
taskId String
|
||||
studentName String
|
||||
studentKey String
|
||||
html String
|
||||
css String
|
||||
js String
|
||||
passed Boolean
|
||||
score Float
|
||||
resultJson String
|
||||
createdAt DateTime @default(now())
|
||||
task Task @relation(fields: [taskId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([taskId, studentKey])
|
||||
}
|
||||
@@ -0,0 +1,675 @@
|
||||
import crypto from "node:crypto";
|
||||
import dns from "node:dns/promises";
|
||||
import fs from "node:fs/promises";
|
||||
import net from "node:net";
|
||||
import path from "node:path";
|
||||
import cookieParser from "cookie-parser";
|
||||
import express from "express";
|
||||
import helmet from "helmet";
|
||||
import multer from "multer";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
clearAuthCookies,
|
||||
hasAdmin,
|
||||
hasStudentAccess,
|
||||
normalizeStudentName,
|
||||
requireAdmin,
|
||||
requireStudentAccess,
|
||||
setAdminCookie,
|
||||
setStudentCookie,
|
||||
studentKey
|
||||
} from "./auth.js";
|
||||
import { config } from "./config.js";
|
||||
import { ensureDatabaseSchema, prisma } from "./db.js";
|
||||
import { generateExpectedScreenshot, judgeSubmission } from "./judge.js";
|
||||
import type { CheckDefinition } from "./types.js";
|
||||
|
||||
const upload = multer({
|
||||
storage: multer.memoryStorage(),
|
||||
limits: { fileSize: 5 * 1024 * 1024 }
|
||||
});
|
||||
|
||||
const checkSchema = z.object({
|
||||
type: z.enum(
|
||||
[
|
||||
"selector-exists",
|
||||
"text-contains",
|
||||
"attribute-equals",
|
||||
"css-property",
|
||||
"element-count",
|
||||
"visual-match"
|
||||
],
|
||||
{ errorMap: () => ({ message: "Choose a supported check type." }) }
|
||||
),
|
||||
selector: z.string().optional(),
|
||||
value: z.string().optional(),
|
||||
attribute: z.string().optional(),
|
||||
property: z.string().optional(),
|
||||
count: z.coerce.number({ invalid_type_error: "Enter a valid element count." }).int().nonnegative().optional(),
|
||||
threshold: z.coerce.number({ invalid_type_error: "Enter a valid visual threshold." }).min(0).max(1).optional(),
|
||||
weight: z.coerce.number({ invalid_type_error: "Enter a valid check weight." }).positive("Check weight must be greater than 0.").default(1),
|
||||
message: z.string().trim().min(1, "Add feedback text for this check.")
|
||||
});
|
||||
|
||||
const taskSchema = z.object({
|
||||
title: z.string().trim().min(1, "Add a task title."),
|
||||
descriptionMarkdown: z.string().default(""),
|
||||
groupId: z.string().trim().min(1, "Choose a day group for this task."),
|
||||
order: z.coerce.number({ invalid_type_error: "Enter a valid task order." }).int("Task order must be a whole number.").default(0),
|
||||
visible: z.boolean().default(true),
|
||||
starterHtml: z.string().default(""),
|
||||
starterCss: z.string().default(""),
|
||||
starterJs: z.string().default(""),
|
||||
referenceHtml: z.string().default(""),
|
||||
referenceCss: z.string().default(""),
|
||||
referenceJs: z.string().default(""),
|
||||
visualThreshold: z
|
||||
.coerce.number({ invalid_type_error: "Enter a valid visual threshold." })
|
||||
.min(0, "Visual threshold must be between 0 and 1.")
|
||||
.max(1, "Visual threshold must be between 0 and 1.")
|
||||
.default(0.85),
|
||||
passingScore: z
|
||||
.coerce.number({ invalid_type_error: "Enter a valid passing score." })
|
||||
.min(0, "Passing score must be between 0 and 1.")
|
||||
.max(1, "Passing score must be between 0 and 1.")
|
||||
.default(0.8),
|
||||
checks: z.array(checkSchema).default([]),
|
||||
regenerateExpected: z.boolean().default(true)
|
||||
}).superRefine((task, context) => {
|
||||
task.checks.forEach((check, index) => {
|
||||
const label = `Check ${index + 1}`;
|
||||
const needsSelector = check.type !== "visual-match";
|
||||
const needsValue = ["text-contains", "attribute-equals", "css-property"].includes(check.type);
|
||||
|
||||
if (needsSelector && !check.selector?.trim()) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["checks", index, "selector"],
|
||||
message: `${label}: enter a CSS selector.`
|
||||
});
|
||||
}
|
||||
|
||||
if (needsValue && !check.value?.trim()) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["checks", index, "value"],
|
||||
message: `${label}: enter the expected value.`
|
||||
});
|
||||
}
|
||||
|
||||
if (check.type === "attribute-equals" && !check.attribute?.trim()) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["checks", index, "attribute"],
|
||||
message: `${label}: enter the attribute name.`
|
||||
});
|
||||
}
|
||||
|
||||
if (check.type === "css-property" && !check.property?.trim()) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["checks", index, "property"],
|
||||
message: `${label}: enter the CSS property name.`
|
||||
});
|
||||
}
|
||||
|
||||
if (check.type === "element-count" && check.count === undefined) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["checks", index, "count"],
|
||||
message: `${label}: enter the expected number of elements.`
|
||||
});
|
||||
}
|
||||
|
||||
if (check.type === "visual-match" && check.threshold !== undefined && (check.threshold < 0 || check.threshold > 1)) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["checks", index, "threshold"],
|
||||
message: `${label}: visual threshold must be between 0 and 1.`
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const submissionSchema = z.object({
|
||||
studentName: z.string().min(1).max(120),
|
||||
html: z.string().max(50000),
|
||||
css: z.string().max(50000),
|
||||
js: z.string().max(50000)
|
||||
});
|
||||
|
||||
function publicTask(task: {
|
||||
id: string;
|
||||
title: string;
|
||||
descriptionMarkdown: string;
|
||||
groupId: string;
|
||||
order: number;
|
||||
starterHtml: string;
|
||||
starterCss: string;
|
||||
starterJs: string;
|
||||
expectedScreenshotPath: string | null;
|
||||
visualThreshold: number;
|
||||
passingScore: number;
|
||||
}) {
|
||||
return task;
|
||||
}
|
||||
|
||||
function taskWithChecks(task: { checksJson: string }) {
|
||||
return {
|
||||
...task,
|
||||
checks: parseChecks(task.checksJson),
|
||||
checksJson: undefined
|
||||
};
|
||||
}
|
||||
|
||||
function parseChecks(checksJson: string): CheckDefinition[] {
|
||||
try {
|
||||
const parsed = JSON.parse(checksJson);
|
||||
return Array.isArray(parsed) ? parsed : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function routeParam(value: string | string[] | undefined) {
|
||||
return Array.isArray(value) ? value[0] : value ?? "";
|
||||
}
|
||||
|
||||
function validationError(error: z.ZodError, title = "Please fix the highlighted fields.") {
|
||||
return {
|
||||
error: title,
|
||||
details: error.issues.map((issue) => issue.message)
|
||||
};
|
||||
}
|
||||
|
||||
function isPrivateIp(address: string) {
|
||||
if (net.isIPv4(address)) {
|
||||
const parts = address.split(".").map(Number);
|
||||
return (
|
||||
parts[0] === 10 ||
|
||||
parts[0] === 127 ||
|
||||
(parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) ||
|
||||
(parts[0] === 192 && parts[1] === 168) ||
|
||||
(parts[0] === 169 && parts[1] === 254)
|
||||
);
|
||||
}
|
||||
|
||||
if (net.isIPv6(address)) {
|
||||
return address === "::1" || address.startsWith("fc") || address.startsWith("fd") || address.startsWith("fe80");
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async function assertSafeDownloadUrl(rawUrl: string) {
|
||||
const url = new URL(rawUrl);
|
||||
if (!["http:", "https:"].includes(url.protocol)) {
|
||||
throw new Error("Only http and https image URLs are allowed.");
|
||||
}
|
||||
|
||||
const records = await dns.lookup(url.hostname, { all: true });
|
||||
if (records.some((record) => isPrivateIp(record.address))) {
|
||||
throw new Error("Image URLs must not resolve to private or local addresses.");
|
||||
}
|
||||
}
|
||||
|
||||
function imageExtension(contentType: string) {
|
||||
if (contentType.includes("png")) return "png";
|
||||
if (contentType.includes("jpeg") || contentType.includes("jpg")) return "jpg";
|
||||
if (contentType.includes("webp")) return "webp";
|
||||
if (contentType.includes("gif")) return "gif";
|
||||
return null;
|
||||
}
|
||||
|
||||
async function saveAsset(buffer: Buffer, contentType: string) {
|
||||
const extension = imageExtension(contentType);
|
||||
if (!extension) {
|
||||
throw new Error("Only png, jpg, webp, and gif images are supported.");
|
||||
}
|
||||
|
||||
const fileName = `${crypto.randomUUID()}.${extension}`;
|
||||
const assetPath = path.join(config.generatedDir, "assets", fileName);
|
||||
await fs.mkdir(path.dirname(assetPath), { recursive: true });
|
||||
await fs.writeFile(assetPath, buffer);
|
||||
return `/generated/assets/${fileName}`;
|
||||
}
|
||||
|
||||
async function maybeGenerateExpected(task: {
|
||||
id: string;
|
||||
referenceHtml: string;
|
||||
referenceCss: string;
|
||||
referenceJs: string;
|
||||
}) {
|
||||
if (!task.referenceHtml.trim() && !task.referenceCss.trim() && !task.referenceJs.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return generateExpectedScreenshot(task.id, {
|
||||
html: task.referenceHtml,
|
||||
css: task.referenceCss,
|
||||
js: task.referenceJs
|
||||
});
|
||||
}
|
||||
|
||||
async function seedSampleData() {
|
||||
const count = await prisma.dayGroup.count();
|
||||
if (count > 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const group = await prisma.dayGroup.create({
|
||||
data: { title: "Day 1 - HTML basics", order: 1 }
|
||||
});
|
||||
const task = await prisma.task.create({
|
||||
data: {
|
||||
title: "Create a welcome button",
|
||||
descriptionMarkdown:
|
||||
"Build a tiny page with a heading and a button. The button should say **Start** and use a green background.",
|
||||
groupId: group.id,
|
||||
order: 1,
|
||||
starterHtml: "<h1>Welcome</h1>\n<button>Start</button>",
|
||||
starterCss: "body {\n display: flex;\n flex-direction: column;\n align-items: center;\n justify-content: center;\n}\nbutton {\n padding: 12px 18px;\n}",
|
||||
referenceHtml: "<h1>Welcome</h1>\n<button>Start</button>",
|
||||
referenceCss:
|
||||
"body { display: flex; flex-direction: column; align-items: center; justify-content: center; background: #f7fafc; }\nbutton { padding: 12px 18px; background: rgb(22, 163, 74); color: white; border: 0; border-radius: 6px; }",
|
||||
checksJson: JSON.stringify([
|
||||
{
|
||||
type: "selector-exists",
|
||||
selector: "button",
|
||||
message: "Add a button element."
|
||||
},
|
||||
{
|
||||
type: "text-contains",
|
||||
selector: "button",
|
||||
value: "Start",
|
||||
message: "The button should contain the text Start."
|
||||
},
|
||||
{
|
||||
type: "css-property",
|
||||
selector: "button",
|
||||
property: "background-color",
|
||||
value: "rgb(22, 163, 74)",
|
||||
message: "Make the button background green."
|
||||
},
|
||||
{
|
||||
type: "visual-match",
|
||||
threshold: 0.75,
|
||||
message: "The page should roughly match the reference screenshot."
|
||||
}
|
||||
])
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
const expectedScreenshotPath = await maybeGenerateExpected(task);
|
||||
if (expectedScreenshotPath) {
|
||||
await prisma.task.update({ where: { id: task.id }, data: { expectedScreenshotPath } });
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("Could not generate sample expected screenshot:", error);
|
||||
}
|
||||
}
|
||||
|
||||
export async function createApp() {
|
||||
await fs.mkdir(config.generatedDir, { recursive: true });
|
||||
await ensureDatabaseSchema();
|
||||
await seedSampleData();
|
||||
|
||||
const app = express();
|
||||
app.disable("x-powered-by");
|
||||
app.use(
|
||||
helmet({
|
||||
contentSecurityPolicy: false,
|
||||
crossOriginEmbedderPolicy: false
|
||||
})
|
||||
);
|
||||
app.use(express.json({ limit: "2mb" }));
|
||||
app.use(cookieParser());
|
||||
app.use("/generated", express.static(config.generatedDir));
|
||||
|
||||
app.get("/api/health", (_req, res) => {
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
app.post("/api/auth/admin", (req, res) => {
|
||||
if (req.body?.password !== config.adminPassword) {
|
||||
res.status(401).json({ error: "Invalid admin password." });
|
||||
return;
|
||||
}
|
||||
|
||||
setAdminCookie(res);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
app.post("/api/auth/student", (req, res) => {
|
||||
if (req.body?.password !== config.studentPassword) {
|
||||
res.status(401).json({ error: "Invalid student password." });
|
||||
return;
|
||||
}
|
||||
|
||||
setStudentCookie(res);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
app.post("/api/auth/logout", (_req, res) => {
|
||||
clearAuthCookies(res);
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
app.get("/api/auth/me", (req, res) => {
|
||||
res.json({
|
||||
admin: hasAdmin(req),
|
||||
studentAccess: hasStudentAccess(req)
|
||||
});
|
||||
});
|
||||
|
||||
app.get("/api/groups", requireStudentAccess, async (_req, res) => {
|
||||
const groups = await prisma.dayGroup.findMany({
|
||||
orderBy: [{ order: "asc" }, { createdAt: "asc" }],
|
||||
include: {
|
||||
tasks: {
|
||||
where: { visible: true },
|
||||
orderBy: [{ order: "asc" }, { createdAt: "asc" }],
|
||||
select: {
|
||||
id: true,
|
||||
title: true,
|
||||
descriptionMarkdown: true,
|
||||
groupId: true,
|
||||
order: true,
|
||||
starterHtml: true,
|
||||
starterCss: true,
|
||||
starterJs: true,
|
||||
expectedScreenshotPath: true,
|
||||
visualThreshold: true,
|
||||
passingScore: true
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
res.json(groups);
|
||||
});
|
||||
|
||||
app.get("/api/tasks/:id", requireStudentAccess, async (req, res) => {
|
||||
const taskId = routeParam(req.params.id);
|
||||
const task = await prisma.task.findFirst({
|
||||
where: { id: taskId, visible: true },
|
||||
select: {
|
||||
id: true,
|
||||
title: true,
|
||||
descriptionMarkdown: true,
|
||||
groupId: true,
|
||||
order: true,
|
||||
starterHtml: true,
|
||||
starterCss: true,
|
||||
starterJs: true,
|
||||
expectedScreenshotPath: true,
|
||||
visualThreshold: true,
|
||||
passingScore: true
|
||||
}
|
||||
});
|
||||
if (!task) {
|
||||
res.status(404).json({ error: "Task not found." });
|
||||
return;
|
||||
}
|
||||
|
||||
res.json(publicTask(task));
|
||||
});
|
||||
|
||||
app.get("/api/tasks/:id/submissions", requireStudentAccess, async (req, res) => {
|
||||
const taskId = routeParam(req.params.id);
|
||||
const name = normalizeStudentName(String(req.query.studentName ?? ""));
|
||||
if (!name) {
|
||||
res.json([]);
|
||||
return;
|
||||
}
|
||||
|
||||
const submissions = await prisma.submission.findMany({
|
||||
where: { taskId, studentKey: studentKey(name) },
|
||||
orderBy: { createdAt: "desc" },
|
||||
take: 20
|
||||
});
|
||||
res.json(submissions.map((submission) => ({ ...submission, result: JSON.parse(submission.resultJson) })));
|
||||
});
|
||||
|
||||
app.post("/api/tasks/:id/submissions", requireStudentAccess, async (req, res) => {
|
||||
const taskId = routeParam(req.params.id);
|
||||
const parsed = submissionSchema.safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
res.status(400).json(validationError(parsed.error, "Please fix the submission before sending it."));
|
||||
return;
|
||||
}
|
||||
|
||||
const task = await prisma.task.findFirst({ where: { id: taskId, visible: true } });
|
||||
if (!task) {
|
||||
res.status(404).json({ error: "Task not found." });
|
||||
return;
|
||||
}
|
||||
|
||||
const studentName = normalizeStudentName(parsed.data.studentName);
|
||||
const result = await judgeSubmission({
|
||||
taskId: task.id,
|
||||
checksJson: task.checksJson,
|
||||
expectedScreenshotPath: task.expectedScreenshotPath,
|
||||
visualThreshold: task.visualThreshold,
|
||||
passingScore: task.passingScore,
|
||||
bundle: {
|
||||
html: parsed.data.html,
|
||||
css: parsed.data.css,
|
||||
js: parsed.data.js
|
||||
}
|
||||
});
|
||||
const submission = await prisma.submission.create({
|
||||
data: {
|
||||
taskId: task.id,
|
||||
studentName,
|
||||
studentKey: studentKey(studentName),
|
||||
html: parsed.data.html,
|
||||
css: parsed.data.css,
|
||||
js: parsed.data.js,
|
||||
passed: result.passed,
|
||||
score: result.score,
|
||||
resultJson: JSON.stringify(result)
|
||||
}
|
||||
});
|
||||
res.json({ submission: { ...submission, result }, result });
|
||||
});
|
||||
|
||||
app.get("/api/admin/groups", requireAdmin, async (_req, res) => {
|
||||
const groups = await prisma.dayGroup.findMany({
|
||||
orderBy: [{ order: "asc" }, { createdAt: "asc" }],
|
||||
include: { tasks: { orderBy: [{ order: "asc" }, { createdAt: "asc" }] } }
|
||||
});
|
||||
res.json(groups.map((group) => ({ ...group, tasks: group.tasks.map(taskWithChecks) })));
|
||||
});
|
||||
|
||||
app.post("/api/admin/groups", requireAdmin, async (req, res) => {
|
||||
const parsed = z.object({ title: z.string().min(1), order: z.coerce.number().int().default(0) }).safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
res.status(400).json(validationError(parsed.error, "Please fix the day group before saving it."));
|
||||
return;
|
||||
}
|
||||
const group = await prisma.dayGroup.create({ data: parsed.data });
|
||||
res.json(group);
|
||||
});
|
||||
|
||||
app.put("/api/admin/groups/:id", requireAdmin, async (req, res) => {
|
||||
const groupId = routeParam(req.params.id);
|
||||
const parsed = z.object({ title: z.string().min(1), order: z.coerce.number().int().default(0) }).safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
res.status(400).json(validationError(parsed.error, "Please fix the day group before saving it."));
|
||||
return;
|
||||
}
|
||||
const group = await prisma.dayGroup.update({ where: { id: groupId }, data: parsed.data });
|
||||
res.json(group);
|
||||
});
|
||||
|
||||
app.delete("/api/admin/groups/:id", requireAdmin, async (req, res) => {
|
||||
const groupId = routeParam(req.params.id);
|
||||
await prisma.dayGroup.delete({ where: { id: groupId } });
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
app.post("/api/admin/tasks", requireAdmin, async (req, res) => {
|
||||
const parsed = taskSchema.safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
res.status(400).json(validationError(parsed.error, "Please fix the task before saving it."));
|
||||
return;
|
||||
}
|
||||
|
||||
const { checks, regenerateExpected, ...data } = parsed.data;
|
||||
const task = await prisma.task.create({
|
||||
data: {
|
||||
...data,
|
||||
checksJson: JSON.stringify(checks)
|
||||
}
|
||||
});
|
||||
|
||||
let expectedScreenshotPath = task.expectedScreenshotPath;
|
||||
if (regenerateExpected) {
|
||||
expectedScreenshotPath = await maybeGenerateExpected(task);
|
||||
if (expectedScreenshotPath) {
|
||||
await prisma.task.update({ where: { id: task.id }, data: { expectedScreenshotPath } });
|
||||
}
|
||||
}
|
||||
|
||||
const created = await prisma.task.findUniqueOrThrow({ where: { id: task.id } });
|
||||
res.json(taskWithChecks(created));
|
||||
});
|
||||
|
||||
app.put("/api/admin/tasks/:id", requireAdmin, async (req, res) => {
|
||||
const taskId = routeParam(req.params.id);
|
||||
const parsed = taskSchema.safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
res.status(400).json(validationError(parsed.error, "Please fix the task before saving it."));
|
||||
return;
|
||||
}
|
||||
|
||||
const { checks, regenerateExpected, ...data } = parsed.data;
|
||||
const task = await prisma.task.update({
|
||||
where: { id: taskId },
|
||||
data: {
|
||||
...data,
|
||||
checksJson: JSON.stringify(checks)
|
||||
}
|
||||
});
|
||||
|
||||
if (regenerateExpected) {
|
||||
const expectedScreenshotPath = await maybeGenerateExpected(task);
|
||||
if (expectedScreenshotPath) {
|
||||
await prisma.task.update({ where: { id: task.id }, data: { expectedScreenshotPath } });
|
||||
}
|
||||
}
|
||||
|
||||
const updated = await prisma.task.findUniqueOrThrow({ where: { id: taskId } });
|
||||
res.json(taskWithChecks(updated));
|
||||
});
|
||||
|
||||
app.post("/api/admin/tasks/:id/generate-expected", requireAdmin, async (req, res) => {
|
||||
const taskId = routeParam(req.params.id);
|
||||
const task = await prisma.task.findUnique({ where: { id: taskId } });
|
||||
if (!task) {
|
||||
res.status(404).json({ error: "Task not found." });
|
||||
return;
|
||||
}
|
||||
const expectedScreenshotPath = await maybeGenerateExpected(task);
|
||||
if (!expectedScreenshotPath) {
|
||||
res.status(400).json({ error: "Add reference HTML, CSS, or JS first." });
|
||||
return;
|
||||
}
|
||||
const updated = await prisma.task.update({ where: { id: task.id }, data: { expectedScreenshotPath } });
|
||||
res.json(taskWithChecks(updated));
|
||||
});
|
||||
|
||||
app.delete("/api/admin/tasks/:id", requireAdmin, async (req, res) => {
|
||||
const taskId = routeParam(req.params.id);
|
||||
await prisma.task.delete({ where: { id: taskId } });
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
app.get("/api/admin/submissions", requireAdmin, async (_req, res) => {
|
||||
const submissions = await prisma.submission.findMany({
|
||||
orderBy: { createdAt: "desc" },
|
||||
take: 200,
|
||||
include: { task: { select: { title: true, group: { select: { title: true } } } } }
|
||||
});
|
||||
res.json(submissions.map((submission) => ({ ...submission, result: JSON.parse(submission.resultJson) })));
|
||||
});
|
||||
|
||||
app.post("/api/admin/submissions/:id/reevaluate", requireAdmin, async (req, res) => {
|
||||
const submissionId = routeParam(req.params.id);
|
||||
const submission = await prisma.submission.findUnique({
|
||||
where: { id: submissionId },
|
||||
include: { task: { include: { group: { select: { title: true } } } } }
|
||||
});
|
||||
|
||||
if (!submission) {
|
||||
res.status(404).json({ error: "Submission not found." });
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await judgeSubmission({
|
||||
taskId: submission.taskId,
|
||||
checksJson: submission.task.checksJson,
|
||||
expectedScreenshotPath: submission.task.expectedScreenshotPath,
|
||||
visualThreshold: submission.task.visualThreshold,
|
||||
passingScore: submission.task.passingScore,
|
||||
bundle: {
|
||||
html: submission.html,
|
||||
css: submission.css,
|
||||
js: submission.js
|
||||
}
|
||||
});
|
||||
|
||||
const updated = await prisma.submission.update({
|
||||
where: { id: submission.id },
|
||||
data: {
|
||||
passed: result.passed,
|
||||
score: result.score,
|
||||
resultJson: JSON.stringify(result)
|
||||
},
|
||||
include: { task: { select: { title: true, group: { select: { title: true } } } } }
|
||||
});
|
||||
|
||||
res.json({ ...updated, result });
|
||||
});
|
||||
|
||||
app.post("/api/admin/assets/upload", requireAdmin, upload.single("image"), async (req, res) => {
|
||||
if (!req.file) {
|
||||
res.status(400).json({ error: "Image file is required." });
|
||||
return;
|
||||
}
|
||||
|
||||
const url = await saveAsset(req.file.buffer, req.file.mimetype);
|
||||
res.json({ url, markdown: `` });
|
||||
});
|
||||
|
||||
app.post("/api/admin/assets/download", requireAdmin, async (req, res) => {
|
||||
const parsed = z.object({ url: z.string().url() }).safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
res.status(400).json(validationError(parsed.error, "Enter a valid image URL."));
|
||||
return;
|
||||
}
|
||||
|
||||
await assertSafeDownloadUrl(parsed.data.url);
|
||||
const response = await fetch(parsed.data.url, { redirect: "error" });
|
||||
const contentType = response.headers.get("content-type") ?? "";
|
||||
const length = Number(response.headers.get("content-length") ?? 0);
|
||||
if (!response.ok || length > 5 * 1024 * 1024) {
|
||||
res.status(400).json({ error: "Could not download a valid image under 5 MB." });
|
||||
return;
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
const url = await saveAsset(buffer, contentType);
|
||||
res.json({ url, markdown: `` });
|
||||
});
|
||||
|
||||
if (config.nodeEnv === "production") {
|
||||
app.use(express.static(config.frontendDistDir));
|
||||
app.get("*", (_req, res) => {
|
||||
res.sendFile(path.join(config.frontendDistDir, "index.html"));
|
||||
});
|
||||
}
|
||||
|
||||
return app;
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import crypto from "node:crypto";
|
||||
import type { NextFunction, Request, Response } from "express";
|
||||
import { config } from "./config.js";
|
||||
|
||||
const COOKIE_OPTIONS = {
|
||||
httpOnly: true,
|
||||
sameSite: "lax" as const,
|
||||
secure: config.nodeEnv === "production",
|
||||
maxAge: 1000 * 60 * 60 * 12
|
||||
};
|
||||
|
||||
const ADMIN_COOKIE = "fj_admin";
|
||||
const STUDENT_COOKIE = "fj_student";
|
||||
|
||||
type Role = "admin" | "student";
|
||||
|
||||
function sign(value: string) {
|
||||
return crypto.createHmac("sha256", config.sessionSecret).update(value).digest("base64url");
|
||||
}
|
||||
|
||||
function makeToken(role: Role) {
|
||||
const payload = `${role}.${Date.now()}`;
|
||||
return `${payload}.${sign(payload)}`;
|
||||
}
|
||||
|
||||
function verifyToken(token: string | undefined, role: Role) {
|
||||
if (!token) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const parts = token.split(".");
|
||||
if (parts.length !== 3) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const payload = `${parts[0]}.${parts[1]}`;
|
||||
const expected = sign(payload);
|
||||
if (parts[0] !== role || parts[2].length !== expected.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return crypto.timingSafeEqual(Buffer.from(parts[2]), Buffer.from(expected));
|
||||
}
|
||||
|
||||
export function setAdminCookie(res: Response) {
|
||||
res.cookie(ADMIN_COOKIE, makeToken("admin"), COOKIE_OPTIONS);
|
||||
}
|
||||
|
||||
export function setStudentCookie(res: Response) {
|
||||
res.cookie(STUDENT_COOKIE, makeToken("student"), COOKIE_OPTIONS);
|
||||
}
|
||||
|
||||
export function clearAuthCookies(res: Response) {
|
||||
res.clearCookie(ADMIN_COOKIE);
|
||||
res.clearCookie(STUDENT_COOKIE);
|
||||
}
|
||||
|
||||
export function hasAdmin(req: Request) {
|
||||
return verifyToken(req.cookies?.[ADMIN_COOKIE], "admin");
|
||||
}
|
||||
|
||||
export function hasStudentAccess(req: Request) {
|
||||
return hasAdmin(req) || verifyToken(req.cookies?.[STUDENT_COOKIE], "student");
|
||||
}
|
||||
|
||||
export function requireAdmin(req: Request, res: Response, next: NextFunction) {
|
||||
if (!hasAdmin(req)) {
|
||||
res.status(401).json({ error: "Admin access required." });
|
||||
return;
|
||||
}
|
||||
|
||||
next();
|
||||
}
|
||||
|
||||
export function requireStudentAccess(req: Request, res: Response, next: NextFunction) {
|
||||
if (!hasStudentAccess(req)) {
|
||||
res.status(401).json({ error: "Student access required." });
|
||||
return;
|
||||
}
|
||||
|
||||
next();
|
||||
}
|
||||
|
||||
export function normalizeStudentName(name: string) {
|
||||
return name.trim().replace(/\s+/g, " ");
|
||||
}
|
||||
|
||||
export function studentKey(name: string) {
|
||||
return normalizeStudentName(name).toLocaleLowerCase("en-US");
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
export type CodeBundle = {
|
||||
html: string;
|
||||
css: string;
|
||||
js: string;
|
||||
};
|
||||
|
||||
export function buildHtmlDocument(bundle: CodeBundle) {
|
||||
return `<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<style>
|
||||
*, *::before, *::after { box-sizing: border-box; }
|
||||
body { margin: 0; min-height: 100vh; font-family: Arial, Helvetica, sans-serif; }
|
||||
${bundle.css}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
${bundle.html}
|
||||
<script>
|
||||
window.alert = () => {};
|
||||
window.confirm = () => false;
|
||||
window.prompt = () => null;
|
||||
</script>
|
||||
<script>${bundle.js}</script>
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
export const backendDir = path.resolve(path.dirname(__filename), "..");
|
||||
export const projectDir = path.resolve(backendDir, "..");
|
||||
|
||||
dotenv.config({ path: path.resolve(projectDir, ".env") });
|
||||
dotenv.config({ path: path.resolve(backendDir, ".env") });
|
||||
|
||||
export const config = {
|
||||
nodeEnv: process.env.NODE_ENV ?? "development",
|
||||
port: Number(process.env.PORT ?? 4000),
|
||||
adminPassword: process.env.ADMIN_PASSWORD ?? "change-admin-password",
|
||||
studentPassword: process.env.STUDENT_PASSWORD ?? "change-student-password",
|
||||
sessionSecret: process.env.SESSION_SECRET ?? "dev-session-secret-change-me",
|
||||
publicUrl: process.env.PUBLIC_URL ?? "http://localhost:4000",
|
||||
viewport: {
|
||||
width: Number(process.env.JUDGE_VIEWPORT_WIDTH ?? 800),
|
||||
height: Number(process.env.JUDGE_VIEWPORT_HEIGHT ?? 600)
|
||||
},
|
||||
dataDir: path.resolve(backendDir, "data"),
|
||||
generatedDir: path.resolve(backendDir, "data", "generated"),
|
||||
frontendDistDir: path.resolve(projectDir, "frontend", "dist")
|
||||
};
|
||||
|
||||
export function assertProductionConfig() {
|
||||
if (config.nodeEnv !== "production") {
|
||||
return;
|
||||
}
|
||||
|
||||
const weakValues = new Set([
|
||||
"change-admin-password",
|
||||
"change-student-password",
|
||||
"dev-session-secret-change-me",
|
||||
"change-this-long-random-secret"
|
||||
]);
|
||||
|
||||
for (const [name, value] of [
|
||||
["ADMIN_PASSWORD", config.adminPassword],
|
||||
["STUDENT_PASSWORD", config.studentPassword],
|
||||
["SESSION_SECRET", config.sessionSecret]
|
||||
]) {
|
||||
if (!value || weakValues.has(value)) {
|
||||
throw new Error(`${name} must be set to a strong value in production.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
|
||||
export const prisma = new PrismaClient();
|
||||
|
||||
export async function ensureDatabaseSchema() {
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE TABLE IF NOT EXISTS "DayGroup" (
|
||||
"id" TEXT NOT NULL PRIMARY KEY,
|
||||
"title" TEXT NOT NULL,
|
||||
"order" INTEGER NOT NULL DEFAULT 0,
|
||||
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE TABLE IF NOT EXISTS "Task" (
|
||||
"id" TEXT NOT NULL PRIMARY KEY,
|
||||
"title" TEXT NOT NULL,
|
||||
"descriptionMarkdown" TEXT NOT NULL,
|
||||
"groupId" TEXT NOT NULL,
|
||||
"order" INTEGER NOT NULL DEFAULT 0,
|
||||
"visible" BOOLEAN NOT NULL DEFAULT true,
|
||||
"starterHtml" TEXT NOT NULL DEFAULT '',
|
||||
"starterCss" TEXT NOT NULL DEFAULT '',
|
||||
"starterJs" TEXT NOT NULL DEFAULT '',
|
||||
"referenceHtml" TEXT NOT NULL DEFAULT '',
|
||||
"referenceCss" TEXT NOT NULL DEFAULT '',
|
||||
"referenceJs" TEXT NOT NULL DEFAULT '',
|
||||
"expectedScreenshotPath" TEXT,
|
||||
"visualThreshold" REAL NOT NULL DEFAULT 0.85,
|
||||
"passingScore" REAL NOT NULL DEFAULT 0.8,
|
||||
"checksJson" TEXT NOT NULL DEFAULT '[]',
|
||||
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT "Task_groupId_fkey" FOREIGN KEY ("groupId") REFERENCES "DayGroup" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
`);
|
||||
|
||||
const taskColumns = await prisma.$queryRawUnsafe<Array<{ name: string }>>(`PRAGMA table_info("Task");`);
|
||||
if (!taskColumns.some((column) => column.name === "passingScore")) {
|
||||
await prisma.$executeRawUnsafe(`ALTER TABLE "Task" ADD COLUMN "passingScore" REAL NOT NULL DEFAULT 0.8;`);
|
||||
}
|
||||
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE TABLE IF NOT EXISTS "Submission" (
|
||||
"id" TEXT NOT NULL PRIMARY KEY,
|
||||
"taskId" TEXT NOT NULL,
|
||||
"studentName" TEXT NOT NULL,
|
||||
"studentKey" TEXT NOT NULL,
|
||||
"html" TEXT NOT NULL,
|
||||
"css" TEXT NOT NULL,
|
||||
"js" TEXT NOT NULL,
|
||||
"passed" BOOLEAN NOT NULL,
|
||||
"score" REAL NOT NULL,
|
||||
"resultJson" TEXT NOT NULL,
|
||||
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT "Submission_taskId_fkey" FOREIGN KEY ("taskId") REFERENCES "Task" ("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
`);
|
||||
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE INDEX IF NOT EXISTS "Submission_taskId_studentKey_idx" ON "Submission" ("taskId", "studentKey");
|
||||
`);
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildHtmlDocument } from "./codeDocument.js";
|
||||
import { summarizeWeightedResults } from "./judge.js";
|
||||
|
||||
describe("buildHtmlDocument", () => {
|
||||
it("combines html, css, and js into a browser document", () => {
|
||||
const document = buildHtmlDocument({
|
||||
html: "<button>Start</button>",
|
||||
css: "button { color: red; }",
|
||||
js: "document.body.dataset.ready = 'true';"
|
||||
});
|
||||
|
||||
expect(document).toContain("<button>Start</button>");
|
||||
expect(document).toContain("button { color: red; }");
|
||||
expect(document).toContain("document.body.dataset.ready");
|
||||
});
|
||||
});
|
||||
|
||||
describe("summarizeWeightedResults", () => {
|
||||
it("passes by default when 80 percent of equal-weight checks pass", () => {
|
||||
const summary = summarizeWeightedResults([
|
||||
{ type: "selector-exists", message: "one", passed: true },
|
||||
{ type: "selector-exists", message: "two", passed: true },
|
||||
{ type: "selector-exists", message: "three", passed: true },
|
||||
{ type: "selector-exists", message: "four", passed: true },
|
||||
{ type: "selector-exists", message: "five", passed: false }
|
||||
]);
|
||||
|
||||
expect(summary).toEqual({ score: 0.8, passed: true });
|
||||
});
|
||||
|
||||
it("uses explicit check weights when calculating the score", () => {
|
||||
const summary = summarizeWeightedResults([
|
||||
{ type: "selector-exists", message: "important", passed: false, weight: 3 },
|
||||
{ type: "selector-exists", message: "small", passed: true, weight: 1 }
|
||||
]);
|
||||
|
||||
expect(summary).toEqual({ score: 0.25, passed: false });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,312 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { PNG } from "pngjs";
|
||||
import pixelmatch from "pixelmatch";
|
||||
import { chromium } from "playwright";
|
||||
import { config } from "./config.js";
|
||||
import { buildHtmlDocument, type CodeBundle } from "./codeDocument.js";
|
||||
import type { CheckDefinition, CheckResult, JudgeResult } from "./types.js";
|
||||
|
||||
async function renderScreenshot(bundle: CodeBundle, outputPath: string) {
|
||||
await fs.mkdir(path.dirname(outputPath), { recursive: true });
|
||||
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const context = await browser.newContext({
|
||||
viewport: config.viewport,
|
||||
javaScriptEnabled: true
|
||||
});
|
||||
|
||||
await context.route("**/*", (route) => {
|
||||
const url = route.request().url();
|
||||
if (url.startsWith("data:") || url.startsWith("about:")) {
|
||||
route.continue();
|
||||
return;
|
||||
}
|
||||
|
||||
route.abort();
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
page.setDefaultTimeout(1500);
|
||||
page.on("dialog", (dialog) => dialog.dismiss().catch(() => undefined));
|
||||
await page.setContent(buildHtmlDocument(bundle), {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 2000
|
||||
});
|
||||
await page.waitForTimeout(150);
|
||||
await page.screenshot({ path: outputPath, fullPage: false });
|
||||
return { browser, page };
|
||||
}
|
||||
|
||||
function parseChecks(checksJson: string): CheckDefinition[] {
|
||||
try {
|
||||
const checks = JSON.parse(checksJson);
|
||||
return Array.isArray(checks) ? checks : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function formatPercent(value: number) {
|
||||
return `${(value * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function checkWeight(check: CheckDefinition) {
|
||||
return typeof check.weight === "number" && Number.isFinite(check.weight) && check.weight > 0 ? check.weight : 1;
|
||||
}
|
||||
|
||||
export function summarizeWeightedResults(results: CheckResult[], passingScore = 0.8) {
|
||||
const totalWeight = results.reduce((sum, result) => sum + (result.weight ?? 1), 0);
|
||||
const passedWeight = results.reduce((sum, result) => sum + (result.passed ? result.weight ?? 1 : 0), 0);
|
||||
const score = totalWeight === 0 ? 1 : Number((passedWeight / totalWeight).toFixed(4));
|
||||
|
||||
return {
|
||||
score,
|
||||
passed: score >= passingScore
|
||||
};
|
||||
}
|
||||
|
||||
type Pixel = {
|
||||
r: number;
|
||||
g: number;
|
||||
b: number;
|
||||
a: number;
|
||||
};
|
||||
|
||||
function pixelAt(image: PNG, index: number): Pixel {
|
||||
const offset = index * 4;
|
||||
return {
|
||||
r: image.data[offset],
|
||||
g: image.data[offset + 1],
|
||||
b: image.data[offset + 2],
|
||||
a: image.data[offset + 3]
|
||||
};
|
||||
}
|
||||
|
||||
function colorDistance(a: Pixel, b: Pixel) {
|
||||
const dr = a.r - b.r;
|
||||
const dg = a.g - b.g;
|
||||
const db = a.b - b.b;
|
||||
const da = (a.a - b.a) / 2;
|
||||
return Math.sqrt(dr * dr + dg * dg + db * db + da * da);
|
||||
}
|
||||
|
||||
function cornerBackground(image: PNG) {
|
||||
return pixelAt(image, 0);
|
||||
}
|
||||
|
||||
function contentComparison(actual: PNG, expected: PNG) {
|
||||
const actualBackground = cornerBackground(actual);
|
||||
const expectedBackground = cornerBackground(expected);
|
||||
const contentTolerance = 18;
|
||||
const mismatchTolerance = 30;
|
||||
let contentPixels = 0;
|
||||
let contentMismatches = 0;
|
||||
|
||||
for (let index = 0; index < actual.width * actual.height; index += 1) {
|
||||
const actualPixel = pixelAt(actual, index);
|
||||
const expectedPixel = pixelAt(expected, index);
|
||||
const isExpectedContent = colorDistance(expectedPixel, expectedBackground) > contentTolerance;
|
||||
const isActualContent = colorDistance(actualPixel, actualBackground) > contentTolerance;
|
||||
const isMismatch = colorDistance(actualPixel, expectedPixel) > mismatchTolerance;
|
||||
|
||||
if (isExpectedContent || isActualContent || isMismatch) {
|
||||
contentPixels += 1;
|
||||
if (isMismatch) {
|
||||
contentMismatches += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (contentPixels === 0) {
|
||||
return {
|
||||
similarity: 1,
|
||||
contentPixels,
|
||||
contentMismatches
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
similarity: Number((1 - contentMismatches / contentPixels).toFixed(4)),
|
||||
contentPixels,
|
||||
contentMismatches
|
||||
};
|
||||
}
|
||||
|
||||
async function compareScreenshots(actualPath: string, expectedPath: string, diffPath: string) {
|
||||
const [actualBuffer, expectedBuffer] = await Promise.all([
|
||||
fs.readFile(actualPath),
|
||||
fs.readFile(expectedPath)
|
||||
]);
|
||||
const actual = PNG.sync.read(actualBuffer);
|
||||
const expected = PNG.sync.read(expectedBuffer);
|
||||
|
||||
if (actual.width !== expected.width || actual.height !== expected.height) {
|
||||
return {
|
||||
similarity: 0,
|
||||
diffPath: undefined,
|
||||
details: `Screenshot size ${actual.width}x${actual.height} did not match expected ${expected.width}x${expected.height}.`
|
||||
};
|
||||
}
|
||||
|
||||
const diff = new PNG({ width: actual.width, height: actual.height });
|
||||
const mismatchedPixels = pixelmatch(actual.data, expected.data, diff.data, actual.width, actual.height, {
|
||||
threshold: 0.12
|
||||
});
|
||||
await fs.mkdir(path.dirname(diffPath), { recursive: true });
|
||||
await fs.writeFile(diffPath, PNG.sync.write(diff));
|
||||
|
||||
const totalPixels = actual.width * actual.height;
|
||||
const content = contentComparison(actual, expected);
|
||||
return {
|
||||
similarity: content.similarity,
|
||||
diffPath,
|
||||
details: `${content.contentMismatches} of ${content.contentPixels} content pixels differed. Full screenshot: ${mismatchedPixels} of ${totalPixels} pixels differed.`
|
||||
};
|
||||
}
|
||||
|
||||
export async function generateExpectedScreenshot(taskId: string, bundle: CodeBundle) {
|
||||
const outputPath = path.join(config.generatedDir, "expected", `${taskId}.png`);
|
||||
const { browser } = await renderScreenshot(bundle, outputPath);
|
||||
await browser.close();
|
||||
return path.relative(config.dataDir, outputPath).replaceAll(path.sep, "/");
|
||||
}
|
||||
|
||||
export async function judgeSubmission(options: {
|
||||
taskId: string;
|
||||
checksJson: string;
|
||||
expectedScreenshotPath?: string | null;
|
||||
visualThreshold: number;
|
||||
passingScore?: number;
|
||||
bundle: CodeBundle;
|
||||
}): Promise<JudgeResult> {
|
||||
const submissionScreenshot = path.join(
|
||||
config.generatedDir,
|
||||
"submissions",
|
||||
`${options.taskId}-${Date.now()}.png`
|
||||
);
|
||||
const { browser, page } = await renderScreenshot(options.bundle, submissionScreenshot);
|
||||
const checks = parseChecks(options.checksJson);
|
||||
const results: CheckResult[] = [];
|
||||
let diffScreenshotPath: string | undefined;
|
||||
|
||||
try {
|
||||
for (const check of checks) {
|
||||
if (check.type === "visual-match") {
|
||||
const threshold = check.threshold ?? options.visualThreshold;
|
||||
if (!options.expectedScreenshotPath) {
|
||||
results.push({
|
||||
type: check.type,
|
||||
message: check.message,
|
||||
passed: false,
|
||||
weight: checkWeight(check),
|
||||
details: "No expected screenshot has been generated for this task."
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const expectedPath = path.join(config.dataDir, options.expectedScreenshotPath);
|
||||
const diffPath = path.join(config.generatedDir, "diffs", `${options.taskId}-${Date.now()}.png`);
|
||||
const comparison = await compareScreenshots(submissionScreenshot, expectedPath, diffPath);
|
||||
if (comparison.diffPath) {
|
||||
diffScreenshotPath = path.relative(config.dataDir, comparison.diffPath).replaceAll(path.sep, "/");
|
||||
}
|
||||
results.push({
|
||||
type: check.type,
|
||||
message: check.message,
|
||||
passed: comparison.similarity >= threshold,
|
||||
score: comparison.similarity,
|
||||
weight: checkWeight(check),
|
||||
details: `${formatPercent(comparison.similarity)} match required ${formatPercent(threshold)}. ${comparison.details}`
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!check.selector) {
|
||||
results.push({
|
||||
type: check.type,
|
||||
message: check.message,
|
||||
passed: false,
|
||||
weight: checkWeight(check),
|
||||
details: "This check is missing a CSS selector."
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const locator = page.locator(check.selector);
|
||||
const count = await locator.count();
|
||||
|
||||
if (check.type === "selector-exists") {
|
||||
results.push({
|
||||
type: check.type,
|
||||
message: check.message,
|
||||
passed: count > 0,
|
||||
weight: checkWeight(check),
|
||||
details: count > 0 ? `Found ${count} matching element(s).` : "No matching element was found."
|
||||
});
|
||||
}
|
||||
|
||||
if (check.type === "element-count") {
|
||||
const expectedCount = check.count ?? Number(check.value ?? 0);
|
||||
results.push({
|
||||
type: check.type,
|
||||
message: check.message,
|
||||
passed: count === expectedCount,
|
||||
weight: checkWeight(check),
|
||||
details: `Found ${count}, expected ${expectedCount}.`
|
||||
});
|
||||
}
|
||||
|
||||
if (check.type === "text-contains") {
|
||||
const text = count > 0 ? await locator.first().textContent() : "";
|
||||
const expected = check.value ?? "";
|
||||
results.push({
|
||||
type: check.type,
|
||||
message: check.message,
|
||||
passed: Boolean(text?.toLocaleLowerCase("en-US").includes(expected.toLocaleLowerCase("en-US"))),
|
||||
weight: checkWeight(check),
|
||||
details: `Expected text containing "${expected}".`
|
||||
});
|
||||
}
|
||||
|
||||
if (check.type === "attribute-equals") {
|
||||
const actual = count > 0 ? await locator.first().getAttribute(check.attribute ?? "") : null;
|
||||
const expected = check.value ?? "";
|
||||
results.push({
|
||||
type: check.type,
|
||||
message: check.message,
|
||||
passed: actual === expected,
|
||||
weight: checkWeight(check),
|
||||
details: `Expected ${check.attribute}="${expected}", found ${actual ?? "nothing"}.`
|
||||
});
|
||||
}
|
||||
|
||||
if (check.type === "css-property") {
|
||||
const actual = count > 0
|
||||
? await locator.first().evaluate((el, property) => getComputedStyle(el).getPropertyValue(property), check.property ?? "")
|
||||
: "";
|
||||
const expected = check.value ?? "";
|
||||
results.push({
|
||||
type: check.type,
|
||||
message: check.message,
|
||||
passed: actual.trim() === expected.trim(),
|
||||
weight: checkWeight(check),
|
||||
details: `Expected ${check.property}: ${expected}, found ${actual || "nothing"}.`
|
||||
});
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
const passingScore = options.passingScore ?? 0.8;
|
||||
const summary = summarizeWeightedResults(results, passingScore);
|
||||
return {
|
||||
passed: summary.passed,
|
||||
score: summary.score,
|
||||
passingScore,
|
||||
results,
|
||||
screenshotPath: path.relative(config.dataDir, submissionScreenshot).replaceAll(path.sep, "/"),
|
||||
diffScreenshotPath
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { assertProductionConfig, config } from "./config.js";
|
||||
import { createApp } from "./app.js";
|
||||
|
||||
assertProductionConfig();
|
||||
|
||||
const app = await createApp();
|
||||
|
||||
app.listen(config.port, () => {
|
||||
console.log(`Frontend Judge listening on http://localhost:${config.port}`);
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
export type CheckType =
|
||||
| "selector-exists"
|
||||
| "text-contains"
|
||||
| "attribute-equals"
|
||||
| "css-property"
|
||||
| "element-count"
|
||||
| "visual-match";
|
||||
|
||||
export type CheckDefinition = {
|
||||
id?: string;
|
||||
type: CheckType;
|
||||
selector?: string;
|
||||
value?: string;
|
||||
attribute?: string;
|
||||
property?: string;
|
||||
count?: number;
|
||||
threshold?: number;
|
||||
weight?: number;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type CheckResult = {
|
||||
type: CheckType;
|
||||
message: string;
|
||||
passed: boolean;
|
||||
details?: string;
|
||||
score?: number;
|
||||
weight?: number;
|
||||
};
|
||||
|
||||
export type JudgeResult = {
|
||||
passed: boolean;
|
||||
score: number;
|
||||
passingScore: number;
|
||||
results: CheckResult[];
|
||||
screenshotPath?: string;
|
||||
diffScreenshotPath?: string;
|
||||
};
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"skipLibCheck": true,
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
Reference in New Issue
Block a user