Skip to content

Commit d92cf4a

Browse files
ojeytonwilliamsojeytonwilliams
authored
feat: drop support for object form python tests (#401)
1 parent 909d72e commit d92cf4a

3 files changed

Lines changed: 28 additions & 178 deletions

File tree

‎packages/python-evaluator/src/python-test-evaluator.ts‎

Lines changed: 9 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,6 @@ import { format } from "../../shared/src/format";
2121
import { ProxyConsole } from "../../shared/src/proxy-console";
2222
import { createAsyncIife } from "../../shared/src/async-iife";
2323

24-
type EvaluatedTeststring = {
25-
input?: string[];
26-
test: () => Promise<unknown>;
27-
};
28-
2924
const READY_MESSAGE: ReadyEvent["data"] = { type: "ready" };
3025

3126
function isProxy(raw: unknown): raw is PyProxy {
@@ -90,102 +85,23 @@ class PythonTestEvaluator implements TestEvaluator {
9085

9186
try {
9287
eval(opts.hooks?.beforeEach ?? "");
93-
// Eval test string to get the dummy input and actual test
94-
const evaluatedTestString = await new Promise<unknown>(
95-
(resolve, reject) => {
96-
try {
97-
const test: unknown = eval(testString);
98-
resolve(test);
99-
} catch (err) {
100-
const isUsingTopLevelAwait =
101-
err instanceof SyntaxError &&
102-
err.message.includes(
103-
"await is only valid in async functions and the top level bodies of modules",
104-
);
105-
106-
if (isUsingTopLevelAwait) {
107-
const iifeTest = createAsyncIife(testString);
108-
109-
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
110-
eval(iifeTest).then(resolve).catch(reject);
111-
} else {
112-
// The assumption is that if we're in this block, then the test
113-
// is EITHER a standard JS test (in which case it should reject)
114-
// OR a Python test that uses the `input()` function (in which case
115-
// we need to fake the input function).
116-
117-
// While supporting both `{ test: () => { // test code } }` and
118-
// `// test code /` we have to fake input before running the
119-
// test. Otherwise any user code that uses `input()` will throw
120-
// an error.
121-
runPython(`
122-
def __fake_input(arg=None):
123-
return ""
12488

125-
input = __fake_input
126-
`);
127-
128-
// Evaluates the learner's code so that any variables they
129-
// define are available to the test.
130-
try {
131-
// It looks the source might be evaluated twice, but they're
132-
// on separate branches.
133-
runPython(opts.source ?? "");
134-
} catch (sourceErr) {
135-
// eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors
136-
reject(sourceErr);
137-
return;
138-
}
139-
140-
try {
141-
eval(testString);
142-
} catch (testErr) {
143-
// eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors
144-
reject(testErr);
145-
}
146-
147-
resolve(undefined);
148-
}
149-
}
150-
},
151-
);
152-
153-
// If the test string does not evaluate to an object, then we assume that
154-
// it's a standard JS test and any assertions have already passed.
155-
if (typeof evaluatedTestString !== "object") {
156-
// Execute afterEach hook if it exists
157-
if (opts.hooks?.afterEach) eval(opts.hooks.afterEach);
158-
159-
return { pass: true, ...this.#proxyConsole.flush() };
160-
}
161-
162-
if (!evaluatedTestString || !("test" in evaluatedTestString)) {
163-
throw Error(
164-
"Test string did not evaluate to an object with the 'test' property",
165-
);
166-
}
167-
168-
const { input, test } = evaluatedTestString as EvaluatedTeststring;
89+
const iifeTest = createAsyncIife(testString);
16990

91+
// If input isn't reassigned, it will throw when called during testing.
17092
runPython(`
171-
def __inputGen(xs):
172-
def gen():
173-
for x in xs:
174-
yield x
175-
iter = gen()
176-
def input(arg=None):
177-
return next(iter)
93+
def __fake_input(arg=None):
94+
return ""
17895
179-
return input
96+
input = __fake_input
97+
`);
18098

181-
input = __inputGen(${JSON.stringify(input ?? [])})
182-
`);
99+
// Evaluates the learner's code so that any variables they
100+
// define are available to the test.
183101

184-
// Evaluates the learner's code so that any variables they define are
185-
// available to the test.
186102
runPython(opts.source ?? "");
187103

188-
await test();
104+
await eval(iifeTest);
189105

190106
if (opts.hooks?.afterEach) eval(opts.hooks.afterEach);
191107

‎packages/tests/integration-tests/index.test.ts‎

Lines changed: 16 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1184,11 +1184,7 @@ assert(mocked.find('.greeting').length === 1);
11841184
type: "python",
11851185
source,
11861186
});
1187-
return runner?.runTest(
1188-
`({
1189-
test: () => assert.equal(runPython('get_five()'), 5),
1190-
})`,
1191-
);
1187+
return runner?.runTest(`assert.equal(runPython('get_five()'), 5)`);
11921188
}, source);
11931189

11941190
expect(result).toEqual({ pass: true });
@@ -1207,11 +1203,7 @@ test: () => assert.equal(runPython('get_five()'), 5),
12071203
const result = await page.evaluate(async () => {
12081204
const runner = window.FCCTestRunner.getRunner("python");
12091205
await runner?.init({}, 1000);
1210-
return runner?.runTest(
1211-
`({
1212-
test: () => assert.equal(runPython('get_five()'), 5),
1213-
})`,
1214-
);
1206+
return runner?.runTest(`assert.equal(runPython('get_five()'), 5)`);
12151207
});
12161208

12171209
expect(result).toMatchObject({
@@ -1230,9 +1222,7 @@ test: () => assert.equal(runPython('get_five()'), 5),
12301222
type: "python",
12311223
});
12321224
return runner?.runTest(
1233-
`({
1234-
test: () => assert.equal(runPython('__name__'), '__main__'),
1235-
})`,
1225+
`assert.equal(runPython('__name__'), '__main__')`,
12361226
);
12371227
});
12381228

@@ -1263,45 +1253,6 @@ test: () => assert.equal(runPython('__name__'), '__main__'),
12631253
});
12641254
});
12651255

1266-
it("should reject testStrings that evaluate to an invalid object", async () => {
1267-
const result = await page.evaluate(async () => {
1268-
const runner = await window.FCCTestRunner.createTestRunner({
1269-
type: "python",
1270-
});
1271-
return runner?.runTest(`({ invalid: 'test' })`);
1272-
});
1273-
1274-
expect(result).toEqual({
1275-
err: {
1276-
message:
1277-
"Test string did not evaluate to an object with the 'test' property",
1278-
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
1279-
stack: expect.stringContaining(
1280-
"Error: Test string did not evaluate to an object with the 'test' property",
1281-
),
1282-
},
1283-
});
1284-
});
1285-
1286-
it("should be able to test with mock input", async () => {
1287-
const result = await page.evaluate(async () => {
1288-
const runner = await window.FCCTestRunner.createTestRunner({
1289-
type: "python",
1290-
source: `
1291-
first = input()
1292-
second = input()
1293-
`,
1294-
});
1295-
1296-
return runner?.runTest(`({
1297-
input: ["argle", "bargle"],
1298-
test: () => assert.equal(runPython('first + second'), "arglebargle")
1299-
})`);
1300-
});
1301-
1302-
expect(result).toEqual({ pass: true });
1303-
});
1304-
13051256
it("should make user code available to the python code as the _code variable", async () => {
13061257
const result = await page.evaluate(async () => {
13071258
const runner = await window.FCCTestRunner.createTestRunner({
@@ -1311,9 +1262,9 @@ second = input()
13111262
},
13121263
});
13131264

1314-
return runner?.runTest(`({
1315-
test: () => assert.equal(runPython('_code'), "test = 'value'")
1316-
})`);
1265+
return runner?.runTest(
1266+
`assert.equal(runPython('_code'), "test = 'value'")`,
1267+
);
13171268
});
13181269

13191270
expect(result).toEqual({ pass: true });
@@ -1324,9 +1275,9 @@ second = input()
13241275
const runner = await window.FCCTestRunner.createTestRunner({
13251276
type: "python",
13261277
});
1327-
return runner?.runTest(`({
1328-
test: () => assert.equal(runPython('_Node("x = 1").get_variable("x")'), 1)
1329-
})`);
1278+
return runner?.runTest(
1279+
`assert.equal(runPython('_Node("x = 1").get_variable("x")'), 1)`,
1280+
);
13301281
});
13311282

13321283
expect(result).toEqual({ pass: true });
@@ -1344,9 +1295,7 @@ second = input()
13441295
},
13451296
});
13461297

1347-
return runner?.runTest(`({
1348-
test: () => {}
1349-
})`);
1298+
return runner?.runTest("");
13501299
});
13511300

13521301
expect(result).toEqual({ pass: true });
@@ -1372,9 +1321,7 @@ second = input()
13721321
const runner = await window.FCCTestRunner.createTestRunner({
13731322
type: "python",
13741323
});
1375-
return runner?.runTest(`({
1376-
test: () => assert.equal(runPython('1 + "1"'), 2)
1377-
})`);
1324+
return runner?.runTest(`assert.equal(runPython('1 + "1"'), 2)`);
13781325
});
13791326
expect(result).toEqual({
13801327
err: {
@@ -1400,9 +1347,7 @@ pattern = re.compile('l+')
14001347
type: "python",
14011348
source,
14021349
});
1403-
return runner?.runTest(`({
1404-
test: () => assert.equal(runPython('str(pattern)'), "l+")
1405-
})`);
1350+
return runner?.runTest(`assert.equal(runPython('str(pattern)'), "l+")`);
14061351
}, source);
14071352
expect(result).toEqual({
14081353
err: {
@@ -1429,9 +1374,9 @@ pattern = re.compile('l+')`;
14291374
});
14301375
// Since the comparison includes a PyProxy object, that will be
14311376
// posted back to the caller and cause a DataCloneError
1432-
return runner?.runTest(`
1433-
({ test: () => assert.equal(__userGlobals.get("pattern"), "l+") })
1434-
`);
1377+
return runner?.runTest(
1378+
`assert.equal(__userGlobals.get("pattern"), "l+")`,
1379+
);
14351380
}, source);
14361381

14371382
expect(result).toEqual({
@@ -1476,7 +1421,7 @@ pattern = re.compile('l+')`;
14761421
});
14771422
});
14781423

1479-
it("should use a fake input function in simple tests", async () => {
1424+
it("should not throw when input is called during a test", async () => {
14801425
const result = await page.evaluate(async () => {
14811426
const runner = await window.FCCTestRunner.createTestRunner({
14821427
type: "python",

‎packages/tests/integration-tests/timeout.test.ts‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -86,9 +86,7 @@ def run():
8686
source,
8787
});
8888
const result = await runner?.runTest(
89-
`({
90-
test: () => assert.equal(runPython('loop()'), 1)
91-
})`,
89+
`assert.equal(runPython('loop()'), 1)`,
9290
10,
9391
);
9492
return result;
@@ -102,11 +100,7 @@ def run():
102100
},
103101
source,
104102
});
105-
return runner?.runTest(
106-
`({
107-
test: () => assert.equal(runPython('run()'), 1)
108-
})`,
109-
);
103+
return runner?.runTest(`assert.equal(runPython('run()'), 1)`);
110104
}, source);
111105

112106
expect(timeoutResult).toEqual({
@@ -132,12 +126,7 @@ while True:
132126
},
133127
source,
134128
});
135-
return runner?.runTest(
136-
`({
137-
test: () => assert.equal(runPython('1'), 1)
138-
})`,
139-
10,
140-
);
129+
return runner?.runTest(`assert.equal(runPython('1'), 1)`, 10);
141130
}, source);
142131

143132
expect(result).toEqual({

0 commit comments

Comments
 (0)