test: Automatic mode testing
refactor: Simplified integration test structure
This commit is contained in:
parent
e4d53733d0
commit
25bb964604
|
@ -3,40 +3,23 @@
|
||||||
import LSpec
|
import LSpec
|
||||||
import Pantograph
|
import Pantograph
|
||||||
import Repl
|
import Repl
|
||||||
|
import Test.Common
|
||||||
|
|
||||||
namespace Pantograph.Test.Integration
|
namespace Pantograph.Test.Integration
|
||||||
open Pantograph
|
open Pantograph
|
||||||
|
|
||||||
def subroutine_named_step (name cmd: String) (payload: List (String × Lean.Json))
|
def step { α } [Lean.ToJson α] (cmd: String) (payload: List (String × Lean.Json))
|
||||||
(expected: Lean.Json): MainM LSpec.TestSeq := do
|
(expected: α) (name? : Option String := .none): MainM LSpec.TestSeq := do
|
||||||
let result ← execute { cmd := cmd, payload := Lean.Json.mkObj payload }
|
let payload := Lean.Json.mkObj payload
|
||||||
return LSpec.test name (toString result = toString expected)
|
let name := name?.getD s!"{cmd} {payload.compress}"
|
||||||
def subroutine_step (cmd: String) (payload: List (String × Lean.Json))
|
let result ← execute { cmd, payload }
|
||||||
(expected: Lean.Json): MainM LSpec.TestSeq := subroutine_named_step cmd cmd payload expected
|
return LSpec.test name (toString result = toString (Lean.toJson expected))
|
||||||
|
|
||||||
def subroutine_runner (steps: List (MainM LSpec.TestSeq)): IO LSpec.TestSeq := do
|
abbrev Test := List (MainM LSpec.TestSeq)
|
||||||
-- Setup the environment for execution
|
|
||||||
let env ← Lean.importModules
|
|
||||||
(imports := #[{module := Lean.Name.str .anonymous "Init", runtimeOnly := false }])
|
|
||||||
(opts := {})
|
|
||||||
(trustLevel := 1)
|
|
||||||
let context: Context := {
|
|
||||||
imports := ["Init"]
|
|
||||||
}
|
|
||||||
let coreContext: Lean.Core.Context ← createCoreContext #[]
|
|
||||||
let commands: MainM LSpec.TestSeq :=
|
|
||||||
steps.foldlM (λ suite step => do
|
|
||||||
let result ← step
|
|
||||||
return suite ++ result) LSpec.TestSeq.done
|
|
||||||
try
|
|
||||||
let coreM := commands.run context |>.run' {}
|
|
||||||
return Prod.fst $ (← coreM.toIO coreContext { env := env })
|
|
||||||
catch ex =>
|
|
||||||
return LSpec.check s!"Uncaught IO exception: {ex.toString}" false
|
|
||||||
|
|
||||||
def test_elab : IO LSpec.TestSeq :=
|
def test_elab : Test :=
|
||||||
subroutine_runner [
|
[
|
||||||
subroutine_step "expr.echo"
|
step "expr.echo"
|
||||||
[("expr", .str "λ {α : Sort (u + 1)} => List α"), ("levels", .arr #["u"])]
|
[("expr", .str "λ {α : Sort (u + 1)} => List α"), ("levels", .arr #["u"])]
|
||||||
(Lean.toJson ({
|
(Lean.toJson ({
|
||||||
type := { pp? := .some "{α : Type u} → Type u" },
|
type := { pp? := .some "{α : Type u} → Type u" },
|
||||||
|
@ -44,46 +27,33 @@ def test_elab : IO LSpec.TestSeq :=
|
||||||
}: Protocol.ExprEchoResult)),
|
}: Protocol.ExprEchoResult)),
|
||||||
]
|
]
|
||||||
|
|
||||||
def test_option_modify : IO LSpec.TestSeq :=
|
def test_option_modify : Test :=
|
||||||
let pp? := Option.some "∀ (n : Nat), n + 1 = n.succ"
|
let pp? := Option.some "∀ (n : Nat), n + 1 = n.succ"
|
||||||
let sexp? := Option.some "(:forall n (:c Nat) ((:c Eq) (:c Nat) ((:c HAdd.hAdd) (:c Nat) (:c Nat) (:c Nat) ((:c instHAdd) (:c Nat) (:c instAddNat)) 0 ((:c OfNat.ofNat) (:c Nat) (:lit 1) ((:c instOfNatNat) (:lit 1)))) ((:c Nat.succ) 0)))"
|
let sexp? := Option.some "(:forall n (:c Nat) ((:c Eq) (:c Nat) ((:c HAdd.hAdd) (:c Nat) (:c Nat) (:c Nat) ((:c instHAdd) (:c Nat) (:c instAddNat)) 0 ((:c OfNat.ofNat) (:c Nat) (:lit 1) ((:c instOfNatNat) (:lit 1)))) ((:c Nat.succ) 0)))"
|
||||||
let module? := Option.some "Init.Data.Nat.Basic"
|
let module? := Option.some "Init.Data.Nat.Basic"
|
||||||
let options: Protocol.Options := {}
|
let options: Protocol.Options := {}
|
||||||
subroutine_runner [
|
[
|
||||||
subroutine_step "env.inspect"
|
step "env.inspect" [("name", .str "Nat.add_one")]
|
||||||
[("name", .str "Nat.add_one")]
|
({ type := { pp? }, module? }: Protocol.EnvInspectResult),
|
||||||
(Lean.toJson ({
|
step "options.set" [("printExprAST", .bool true)]
|
||||||
type := { pp? }, module? }:
|
({ }: Protocol.OptionsSetResult),
|
||||||
Protocol.EnvInspectResult)),
|
step "env.inspect" [("name", .str "Nat.add_one")]
|
||||||
subroutine_step "options.set"
|
({ type := { pp?, sexp? }, module? }: Protocol.EnvInspectResult),
|
||||||
[("printExprAST", .bool true)]
|
step "options.print" []
|
||||||
(Lean.toJson ({ }:
|
({ options with printExprAST := true }: Protocol.Options),
|
||||||
Protocol.OptionsSetResult)),
|
|
||||||
subroutine_step "env.inspect"
|
|
||||||
[("name", .str "Nat.add_one")]
|
|
||||||
(Lean.toJson ({
|
|
||||||
type := { pp?, sexp? }, module? }:
|
|
||||||
Protocol.EnvInspectResult)),
|
|
||||||
subroutine_step "options.print"
|
|
||||||
[]
|
|
||||||
(Lean.toJson ({ options with printExprAST := true }:
|
|
||||||
Protocol.Options))
|
|
||||||
]
|
]
|
||||||
def test_malformed_command : IO LSpec.TestSeq :=
|
def test_malformed_command : Test :=
|
||||||
let invalid := "invalid"
|
let invalid := "invalid"
|
||||||
subroutine_runner [
|
[
|
||||||
subroutine_named_step "Invalid command" invalid
|
step invalid [("name", .str "Nat.add_one")]
|
||||||
[("name", .str "Nat.add_one")]
|
({ error := "command", desc := s!"Unknown command {invalid}" }: Protocol.InteractionError)
|
||||||
(Lean.toJson ({
|
(name? := .some "Invalid Command"),
|
||||||
error := "command", desc := s!"Unknown command {invalid}"}:
|
step "expr.echo" [(invalid, .str "Random garbage data")]
|
||||||
Protocol.InteractionError)),
|
({ error := "command", desc := s!"Unable to parse json: Pantograph.Protocol.ExprEcho.expr: String expected" }:
|
||||||
subroutine_named_step "JSON Deserialization" "expr.echo"
|
Protocol.InteractionError)
|
||||||
[(invalid, .str "Random garbage data")]
|
(name? := .some "JSON Deserialization")
|
||||||
(Lean.toJson ({
|
|
||||||
error := "command", desc := s!"Unable to parse json: Pantograph.Protocol.ExprEcho.expr: String expected"}:
|
|
||||||
Protocol.InteractionError))
|
|
||||||
]
|
]
|
||||||
def test_tactic : IO LSpec.TestSeq :=
|
def test_tactic : Test :=
|
||||||
let goal1: Protocol.Goal := {
|
let goal1: Protocol.Goal := {
|
||||||
name := "_uniq.11",
|
name := "_uniq.11",
|
||||||
target := { pp? := .some "∀ (q : Prop), x ∨ q → q ∨ x" },
|
target := { pp? := .some "∀ (q : Prop), x ∨ q → q ∨ x" },
|
||||||
|
@ -97,77 +67,123 @@ def test_tactic : IO LSpec.TestSeq :=
|
||||||
{ name := "_uniq.16", userName := "y", type? := .some { pp? := .some "Prop" }}
|
{ name := "_uniq.16", userName := "y", type? := .some { pp? := .some "Prop" }}
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
subroutine_runner [
|
[
|
||||||
subroutine_step "goal.start"
|
step "goal.start" [("expr", .str "∀ (p q: Prop), p ∨ q → q ∨ p")]
|
||||||
[("expr", .str "∀ (p q: Prop), p ∨ q → q ∨ p")]
|
({ stateId := 0, root := "_uniq.9" }: Protocol.GoalStartResult),
|
||||||
(Lean.toJson ({stateId := 0, root := "_uniq.9"}:
|
step "goal.tactic" [("stateId", .num 0), ("goalId", .num 0), ("tactic", .str "intro x")]
|
||||||
Protocol.GoalStartResult)),
|
({ nextStateId? := .some 1, goals? := #[goal1], }: Protocol.GoalTacticResult),
|
||||||
subroutine_step "goal.tactic"
|
step "goal.print" [("stateId", .num 1)]
|
||||||
[("stateId", .num 0), ("goalId", .num 0), ("tactic", .str "intro x")]
|
({ parent? := .some { pp? := .some "fun x => ?m.12 x" }, }: Protocol.GoalPrintResult),
|
||||||
(Lean.toJson ({
|
step "goal.tactic" [("stateId", .num 1), ("goalId", .num 0), ("tactic", .str "intro y")]
|
||||||
nextStateId? := .some 1,
|
({ nextStateId? := .some 2, goals? := #[goal2], }: Protocol.GoalTacticResult),
|
||||||
goals? := #[goal1],
|
]
|
||||||
}:
|
def test_automatic_mode (automatic: Bool): Test :=
|
||||||
Protocol.GoalTacticResult)),
|
let varsPQ := #[
|
||||||
subroutine_step "goal.print"
|
{ name := "_uniq.10", userName := "p", type? := .some { pp? := .some "Prop" }},
|
||||||
[("stateId", .num 1)]
|
{ name := "_uniq.13", userName := "q", type? := .some { pp? := .some "Prop" }}
|
||||||
(Lean.toJson ({
|
]
|
||||||
parent? := .some { pp? := .some "fun x => ?m.12 x" },
|
let goal1: Protocol.Goal := {
|
||||||
}:
|
name := "_uniq.17",
|
||||||
Protocol.GoalPrintResult)),
|
target := { pp? := .some "q ∨ p" },
|
||||||
subroutine_step "goal.tactic"
|
vars := varsPQ ++ #[
|
||||||
[("stateId", .num 1), ("goalId", .num 0), ("tactic", .str "intro y")]
|
{ name := "_uniq.16", userName := "h", type? := .some { pp? := .some "p ∨ q" }}
|
||||||
(Lean.toJson ({
|
],
|
||||||
nextStateId? := .some 2,
|
}
|
||||||
goals? := #[goal2],
|
let goal2l: Protocol.Goal := {
|
||||||
}:
|
name := "_uniq.59",
|
||||||
Protocol.GoalTacticResult))
|
userName? := .some "inl",
|
||||||
|
target := { pp? := .some "q ∨ p" },
|
||||||
|
vars := varsPQ ++ #[
|
||||||
|
{ name := "_uniq.47", userName := "h✝", type? := .some { pp? := .some "p" }, isInaccessible := true}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
let goal2r: Protocol.Goal := {
|
||||||
|
name := "_uniq.72",
|
||||||
|
userName? := .some "inr",
|
||||||
|
target := { pp? := .some "q ∨ p" },
|
||||||
|
vars := varsPQ ++ #[
|
||||||
|
{ name := "_uniq.60", userName := "h✝", type? := .some { pp? := .some "q" }, isInaccessible := true}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
let goal3l: Protocol.Goal := {
|
||||||
|
name := "_uniq.78",
|
||||||
|
userName? := .some "inl.h",
|
||||||
|
target := { pp? := .some "p" },
|
||||||
|
vars := varsPQ ++ #[
|
||||||
|
{ name := "_uniq.47", userName := "h✝", type? := .some { pp? := .some "p" }, isInaccessible := true}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
[
|
||||||
|
step "options.set" [("automaticMode", .bool automatic)]
|
||||||
|
({}: Protocol.OptionsSetResult),
|
||||||
|
step "goal.start" [("expr", .str "∀ (p q: Prop), p ∨ q → q ∨ p")]
|
||||||
|
({ stateId := 0, root := "_uniq.9" }: Protocol.GoalStartResult),
|
||||||
|
step "goal.tactic" [("stateId", .num 0), ("goalId", .num 0), ("tactic", .str "intro p q h")]
|
||||||
|
({ nextStateId? := .some 1, goals? := #[goal1], }: Protocol.GoalTacticResult),
|
||||||
|
step "goal.tactic" [("stateId", .num 1), ("goalId", .num 0), ("tactic", .str "cases h")]
|
||||||
|
({ nextStateId? := .some 2, goals? := #[goal2l, goal2r], }: Protocol.GoalTacticResult),
|
||||||
|
let goals? := if automatic then #[goal3l, goal2r] else #[goal3l]
|
||||||
|
step "goal.tactic" [("stateId", .num 2), ("goalId", .num 0), ("tactic", .str "apply Or.inr")]
|
||||||
|
({ nextStateId? := .some 3, goals?, }: Protocol.GoalTacticResult),
|
||||||
]
|
]
|
||||||
|
|
||||||
def test_env_add_inspect : IO LSpec.TestSeq :=
|
def test_env_add_inspect : Test :=
|
||||||
let name1 := "Pantograph.mystery"
|
let name1 := "Pantograph.mystery"
|
||||||
let name2 := "Pantograph.mystery2"
|
let name2 := "Pantograph.mystery2"
|
||||||
subroutine_runner [
|
[
|
||||||
subroutine_step "env.add"
|
step "env.add"
|
||||||
[
|
[
|
||||||
("name", .str name1),
|
("name", .str name1),
|
||||||
("type", .str "Prop → Prop → Prop"),
|
("type", .str "Prop → Prop → Prop"),
|
||||||
("value", .str "λ (a b: Prop) => Or a b"),
|
("value", .str "λ (a b: Prop) => Or a b"),
|
||||||
("isTheorem", .bool false)
|
("isTheorem", .bool false)
|
||||||
]
|
]
|
||||||
(Lean.toJson ({}: Protocol.EnvAddResult)),
|
({}: Protocol.EnvAddResult),
|
||||||
subroutine_step "env.inspect"
|
step "env.inspect" [("name", .str name1)]
|
||||||
[("name", .str name1)]
|
({
|
||||||
(Lean.toJson ({
|
|
||||||
value? := .some { pp? := .some "fun a b => a ∨ b" },
|
value? := .some { pp? := .some "fun a b => a ∨ b" },
|
||||||
type := { pp? := .some "Prop → Prop → Prop" },
|
type := { pp? := .some "Prop → Prop → Prop" },
|
||||||
}:
|
}:
|
||||||
Protocol.EnvInspectResult)),
|
Protocol.EnvInspectResult),
|
||||||
subroutine_step "env.add"
|
step "env.add"
|
||||||
[
|
[
|
||||||
("name", .str name2),
|
("name", .str name2),
|
||||||
("type", .str "Nat → Int"),
|
("type", .str "Nat → Int"),
|
||||||
("value", .str "λ (a: Nat) => a + 1"),
|
("value", .str "λ (a: Nat) => a + 1"),
|
||||||
("isTheorem", .bool false)
|
("isTheorem", .bool false)
|
||||||
]
|
]
|
||||||
(Lean.toJson ({}: Protocol.EnvAddResult)),
|
({}: Protocol.EnvAddResult),
|
||||||
subroutine_step "env.inspect"
|
step "env.inspect" [("name", .str name2)]
|
||||||
[("name", .str name2)]
|
({
|
||||||
(Lean.toJson ({
|
|
||||||
value? := .some { pp? := .some "fun a => ↑a + 1" },
|
value? := .some { pp? := .some "fun a => ↑a + 1" },
|
||||||
type := { pp? := .some "Nat → Int" },
|
type := { pp? := .some "Nat → Int" },
|
||||||
}:
|
}:
|
||||||
Protocol.EnvInspectResult))
|
Protocol.EnvInspectResult)
|
||||||
]
|
]
|
||||||
|
|
||||||
def suite: List (String × IO LSpec.TestSeq) :=
|
def runTest (env: Lean.Environment) (steps: Test): IO LSpec.TestSeq := do
|
||||||
[
|
-- Setup the environment for execution
|
||||||
("Elab", test_elab),
|
let context: Context := {
|
||||||
("Option modify", test_option_modify),
|
imports := ["Init"]
|
||||||
|
}
|
||||||
|
let commands: MainM LSpec.TestSeq :=
|
||||||
|
steps.foldlM (λ suite step => do
|
||||||
|
let result ← step
|
||||||
|
return suite ++ result) LSpec.TestSeq.done
|
||||||
|
runCoreMSeq env <| commands.run context |>.run' {}
|
||||||
|
|
||||||
|
|
||||||
|
def suite (env : Lean.Environment): List (String × IO LSpec.TestSeq) :=
|
||||||
|
let tests := [
|
||||||
|
("expr.echo", test_elab),
|
||||||
|
("options.set options.print", test_option_modify),
|
||||||
("Malformed command", test_malformed_command),
|
("Malformed command", test_malformed_command),
|
||||||
("Tactic", test_tactic),
|
("Tactic", test_tactic),
|
||||||
|
("Manual Mode", test_automatic_mode false),
|
||||||
|
("Automatic Mode", test_automatic_mode true),
|
||||||
("env.add env.inspect", test_env_add_inspect),
|
("env.add env.inspect", test_env_add_inspect),
|
||||||
]
|
]
|
||||||
|
tests.map (fun (name, test) => (name, runTest env test))
|
||||||
|
|
||||||
|
|
||||||
end Pantograph.Test.Integration
|
end Pantograph.Test.Integration
|
||||||
|
|
|
@ -44,7 +44,7 @@ def main (args: List String) := do
|
||||||
|
|
||||||
let suites: List (String × List (String × IO LSpec.TestSeq)) := [
|
let suites: List (String × List (String × IO LSpec.TestSeq)) := [
|
||||||
("Environment", Environment.suite),
|
("Environment", Environment.suite),
|
||||||
("Integration", Integration.suite),
|
("Integration", Integration.suite env_default),
|
||||||
("Library", Library.suite env_default),
|
("Library", Library.suite env_default),
|
||||||
("Metavar", Metavar.suite env_default),
|
("Metavar", Metavar.suite env_default),
|
||||||
("Proofs", Proofs.suite env_default),
|
("Proofs", Proofs.suite env_default),
|
||||||
|
|
Loading…
Reference in New Issue