Merge pull request 'test(frontend): Circular sorry' (#244) from frontend/circular into dev

Reviewed-on: #244

Main.lean

@@ -8,6 +8,12 @@ import Repl
 open Pantograph.Repl
 open Pantograph.Protocol
 
+/-- Print a string to stdout without buffering -/
+def printImmediate (s : String) : IO Unit := do
+  let stdout ← IO.getStdout
+  stdout.putStr (s ++ "\n")
+  stdout.flush
+
 /-- Parse a command either in `{ "cmd": ..., "payload": ... }` form or `cmd { ... }` form. -/
 def parseCommand (s: String): Except String Command := do
   match s.trim.get? 0 with
@@ -34,37 +40,36 @@ partial def loop : MainM Unit := do repeat do
   | .error error =>
     let error  := Lean.toJson ({ error := "command", desc := error }: InteractionError)
     -- Using `Lean.Json.compress` here to prevent newline
-    IO.println error.compress
+    printImmediate error.compress
   | .ok command =>
     try
       let ret ← execute command
       let str := match state.options.printJsonPretty with
         | true => ret.pretty
         | false => ret.compress
-      IO.println str
+      printImmediate str
     catch e =>
-      let message ← e.toMessageData.toString
+      let message := e.toString
       let error  := Lean.toJson ({ error := "main", desc := message }: InteractionError)
-      IO.println error.compress
+      printImmediate error.compress
 
-
-unsafe def main (args: List String): IO Unit := do
+def main (args: List String): IO Unit := do
   -- NOTE: A more sophisticated scheme of command line argument handling is needed.
   if args == ["--version"] then do
     IO.println s!"{Pantograph.version}"
     return
 
-  Pantograph.initSearch ""
+  unsafe do
+    Pantograph.initSearch ""
 
   -- Separate imports and options
   let (options, imports) := args.partition (·.startsWith "--")
   let coreContext ← options.map (·.drop 2) |>.toArray |> Pantograph.createCoreContext
   let coreState ← Pantograph.createCoreState imports.toArray
-  let context: Context := {}
   try
-    let coreM := loop.run context |>.run' {}
-    IO.println "ready."
-    discard <| coreM.toIO coreContext coreState
+    let mainM := loop.run { coreContext } |>.run' { env := coreState.env }
+    printImmediate "ready."
+    mainM
   catch ex =>
     let message := ex.toString
     let error  := Lean.toJson ({ error := "io", desc := message }: InteractionError)

Pantograph/Delate.lean

@@ -26,7 +26,7 @@ def analyzeProjection (env: Environment) (e: Expr): Projection :=
     | _ => panic! "Argument must be proj"
   if (getStructureInfo? env typeName).isSome then
     let ctor := getStructureCtor env typeName
-    let fieldName := getStructureFields env typeName |>.get! idx
+    let fieldName := (getStructureFields env typeName)[idx]!
     let projector := getProjFnForField? env typeName fieldName |>.get!
     .field projector ctor.numParams
   else
@@ -37,7 +37,7 @@ def analyzeProjection (env: Environment) (e: Expr): Projection :=
 def anonymousLevel : Level := .mvar ⟨.anonymous⟩
 
 @[export pantograph_expr_proj_to_app]
-def exprProjToApp (env: Environment) (e: Expr): Expr :=
+def exprProjToApp (env : Environment) (e : Expr) : Expr :=
   let anon : Expr := .mvar ⟨.anonymous⟩
   match analyzeProjection env e with
   | .field projector numParams =>
@@ -63,12 +63,25 @@ def exprProjToApp (env: Environment) (e: Expr): Expr :=
       (List.range numFields)
     mkAppN callee (typeArgs ++ [motive, major, induct]).toArray
 
-def _root_.Lean.Name.isAuxLemma (n : Lean.Name) : Bool := n matches .num (.str _ "_auxLemma") _
+def isAuxLemma (n : Name) : Bool :=
+  match n with
+  -- `mkAuxLemma` generally allows for arbitrary prefixes but these are the ones produced by core.
+  | .str _ s => "_proof_".isPrefixOf s || "_simp_".isPrefixOf s
+  | _ => false
 
 /-- Unfold all lemmas created by `Lean.Meta.mkAuxLemma`. These end in `_auxLemma.nn` where `nn` is a number. -/
-@[export pantograph_unfold_aux_lemmas]
+@[export pantograph_unfold_aux_lemmas_m]
 def unfoldAuxLemmas (e : Expr) : CoreM Expr := do
-  Lean.Meta.deltaExpand e Lean.Name.isAuxLemma
+  Meta.deltaExpand e isAuxLemma
+/-- Unfold all matcher applications -/
+@[export pantograph_unfold_matchers_m]
+def unfoldMatchers (expr : Expr) : CoreM Expr :=
+  Core.transform expr λ e => do
+    let .some mapp ← Meta.matchMatcherApp? e | return .continue e
+    let .some matcherInfo := (← getEnv).find? mapp.matcherName | panic! "Matcher must exist"
+    let f ← Meta.instantiateValueLevelParams matcherInfo mapp.matcherLevels.toList
+    let mdata := KVMap.empty.insert `matcher (DataValue.ofName mapp.matcherName)
+    return .visit $ .mdata mdata (f.betaRev e.getAppRevArgs (useZeta := true))
 
 /--
 Force the instantiation of delayed metavariables even if they cannot be fully
@@ -85,91 +98,93 @@ This function ensures any metavariable in the result is either
 1. Delayed assigned with its pending mvar not assigned in any form
 2. Not assigned (delay or not)
  -/
-partial def instantiateDelayedMVars (eOrig: Expr) : MetaM Expr := do
-  --let padding := String.join $ List.replicate level "│ "
-  --IO.println s!"{padding}Starting {toString eOrig}"
-  let mut result ← Meta.transform (← instantiateMVars eOrig)
-    (pre := fun e => e.withApp fun f args => do
-      let .mvar mvarId := f | return .continue
-      --IO.println s!"{padding}├V {e}"
-      let mvarDecl ← mvarId.getDecl
+partial def instantiateDelayedMVars (expr : Expr) : MetaM Expr :=
+  withTraceNode `Pantograph.Delate (λ _ex => return m!":= {← Meta.ppExpr expr}") do
+  let mut result ← Meta.transform (← instantiateMVars expr)
+    λ e => e.withApp fun f args => do
+    let .mvar mvarId := f | return .continue
+    trace[Pantograph.Delate] "V {e}"
+    let mvarDecl ← mvarId.getDecl
 
-      -- This is critical to maintaining the interdependency of metavariables.
-      -- Without setting `.syntheticOpaque`, Lean's metavariable elimination
-      -- system will not make the necessary delayed assigned mvars in case of
-      -- nested mvars.
-      mvarId.setKind .syntheticOpaque
+    -- This is critical to maintaining the interdependency of metavariables.
+    -- Without setting `.syntheticOpaque`, Lean's metavariable elimination
+    -- system will not make the necessary delayed assigned mvars in case of
+    -- nested mvars.
+    mvarId.setKind .syntheticOpaque
 
-      mvarId.withContext do
-        let lctx ← MonadLCtx.getLCtx
-        if mvarDecl.lctx.any (λ decl => !lctx.contains decl.fvarId) then
-          let violations := mvarDecl.lctx.decls.foldl (λ acc decl? => match decl? with
-            | .some decl => if lctx.contains decl.fvarId then acc else acc ++ [decl.fvarId.name]
-            | .none => acc) []
-          panic! s!"In the context of {mvarId.name}, there are local context variable violations: {violations}"
+    mvarId.withContext do
+      let lctx ← MonadLCtx.getLCtx
+      if mvarDecl.lctx.any (λ decl => !lctx.contains decl.fvarId) then
+        let violations := mvarDecl.lctx.decls.foldl (λ acc decl? => match decl? with
+          | .some decl => if lctx.contains decl.fvarId then acc else acc ++ [decl.fvarId.name]
+          | .none => acc) []
+        panic! s!"In the context of {mvarId.name}, there are local context variable violations: {violations}"
 
-        if let .some assign ← getExprMVarAssignment? mvarId then
-          --IO.println s!"{padding}├A ?{mvarId.name}"
-          assert! !(← mvarId.isDelayedAssigned)
-          return .visit (mkAppN assign args)
-        else if let some { fvars, mvarIdPending } ← getDelayedMVarAssignment? mvarId then
-          --let substTableStr := String.intercalate ", " $ Array.zipWith fvars args (λ fvar assign => s!"{fvar.fvarId!.name} := {assign}") |>.toList
-          --IO.println s!"{padding}├MD ?{mvarId.name} := ?{mvarIdPending.name} [{substTableStr}]"
+      if let .some assign ← getExprMVarAssignment? mvarId then
+        trace[Pantograph.Delate] "A ?{mvarId.name}"
+        assert! !(← mvarId.isDelayedAssigned)
+        return .visit (mkAppN assign args)
+      else if let some { fvars, mvarIdPending } ← getDelayedMVarAssignment? mvarId then
+        if ← isTracingEnabledFor `Pantograph.Delate then
+          let substTableStr := ",".intercalate $
+            Array.zipWith (λ fvar assign => s!"{fvar.fvarId!.name} := {assign}") fvars args |>.toList
+          trace[Pantograph.Delate]"MD ?{mvarId.name} := ?{mvarIdPending.name} [{substTableStr}]"
 
-          if args.size < fvars.size then
-            throwError "Not enough arguments to instantiate a delay assigned mvar. This is due to bad implementations of a tactic: {args.size} < {fvars.size}. Expr: {toString e}; Origin: {toString eOrig}"
-          --if !args.isEmpty then
-            --IO.println s!"{padding}├── Arguments Begin"
-          let args ← args.mapM self
-          --if !args.isEmpty then
-            --IO.println s!"{padding}├── Arguments End"
-          if !(← mvarIdPending.isAssignedOrDelayedAssigned) then
-            --IO.println s!"{padding}├T1"
-            let result := mkAppN f args
-            return .done result
-
-          let pending ← mvarIdPending.withContext do
-            let inner ← instantiateDelayedMVars (.mvar mvarIdPending) --(level := level + 1)
-            --IO.println s!"{padding}├Pre: {inner}"
-            pure <| (← inner.abstractM fvars).instantiateRev args
-
-          -- Tail arguments
-          let result := mkAppRange pending fvars.size args.size args
-          --IO.println s!"{padding}├MD {result}"
+        if args.size < fvars.size then
+          throwError "Not enough arguments to instantiate a delay assigned mvar. This is due to bad implementations of a tactic: {args.size} < {fvars.size}. Expr: {toString e}; Origin: {toString expr}"
+        if !args.isEmpty then
+          trace[Pantograph.Delate] "─ Arguments Begin"
+        let args ← args.mapM self
+        if !args.isEmpty then
+          trace[Pantograph.Delate] "─ Arguments End"
+        if !(← mvarIdPending.isAssignedOrDelayedAssigned) then
+          trace[Pantograph.Delate] "T1"
+          let result := mkAppN f args
           return .done result
-        else
-          assert! !(← mvarId.isAssigned)
-          assert! !(← mvarId.isDelayedAssigned)
-          --if !args.isEmpty then
-          --  IO.println s!"{padding}├── Arguments Begin"
-          let args ← args.mapM self
-          --if !args.isEmpty then
-          --  IO.println s!"{padding}├── Arguments End"
 
-          --IO.println s!"{padding}├M ?{mvarId.name}"
-          return .done (mkAppN f args))
-  --IO.println s!"{padding}└Result {result}"
+        let pending ← mvarIdPending.withContext do
+          let inner ← instantiateDelayedMVars (.mvar mvarIdPending)
+          trace[Pantograph.Delate] "Pre: {inner}"
+          pure <| (← inner.abstractM fvars).instantiateRev args
+
+        -- Tail arguments
+        let result := mkAppRange pending fvars.size args.size args
+        trace[Pantograph.Delate] "MD {result}"
+        return .done result
+      else
+        assert! !(← mvarId.isAssigned)
+        assert! !(← mvarId.isDelayedAssigned)
+        if !args.isEmpty then
+          trace[Pantograph.Delate] "─ Arguments Begin"
+        let args ← args.mapM self
+        if !args.isEmpty then
+          trace[Pantograph.Delate] "─ Arguments End"
+
+        trace[Pantograph.Delate] "M ?{mvarId.name}"
+        return .done (mkAppN f args)
+  trace[Pantoraph.Delate] "Result {result}"
   return result
   where
-  self e := instantiateDelayedMVars e --(level := level + 1)
+  self e := instantiateDelayedMVars e
 
 /--
-Convert an expression to an equiavlent form with
+Convert an expression to an equivalent form with
 1. No nested delayed assigned mvars
-2. No aux lemmas
+2. No aux lemmas or matchers
 3. No assigned mvars
  -/
 @[export pantograph_instantiate_all_m]
-def instantiateAll (e: Expr): MetaM Expr := do
+def instantiateAll (e : Expr) : MetaM Expr := do
   let e ← instantiateDelayedMVars e
   let e ← unfoldAuxLemmas e
+  let e ← unfoldMatchers e
   return e
 
 structure DelayedMVarInvocation where
-  mvarIdPending: MVarId
-  args: Array (FVarId × (Option Expr))
+  mvarIdPending : MVarId
+  args : Array (FVarId × (Option Expr))
   -- Extra arguments applied to the result of this substitution
-  tail: Array Expr
+  tail : Array Expr
 
 -- The pending mvar of any delayed assigned mvar must not be assigned in any way.
 @[export pantograph_to_delayed_mvar_invocation_m]
@@ -440,7 +455,6 @@ def serializeExpression (options: @&Protocol.Options) (e: Expr): MetaM Protocol.
     dependentMVars?,
   }
 
-
 /-- Adapted from ppGoal -/
 def serializeGoal (options: @&Protocol.Options) (goal: MVarId) (mvarDecl: MetavarDecl) (parentDecl?: Option MetavarDecl := .none)
       : MetaM Protocol.Goal := do
@@ -506,7 +520,6 @@ def serializeGoal (options: @&Protocol.Options) (goal: MVarId) (mvarDecl: Metava
     return {
       name := goal.name.toString,
       userName? := if mvarDecl.userName == .anonymous then .none else .some (ofName mvarDecl.userName),
-      isConversion := isLHSGoal? mvarDecl.type |>.isSome,
       target := (← serializeExpression options (← instantiate mvarDecl.type)),
       vars := vars.reverse.toArray
     }
@@ -516,17 +529,20 @@ def serializeGoal (options: @&Protocol.Options) (goal: MVarId) (mvarDecl: Metava
 
 protected def GoalState.serializeGoals
       (state: GoalState)
-      (parent: Option GoalState := .none)
       (options: @&Protocol.Options := {}):
     MetaM (Array Protocol.Goal):= do
   state.restoreMetaM
   let goals := state.goals.toArray
-  let parentDecl? := parent.bind (λ parentState => parentState.mctx.findDecl? state.parentMVar?.get!)
   goals.mapM fun goal => do
+    let fragment := match state.fragments[goal]? with
+      | .none => .tactic
+      | .some $ .calc .. => .calc
+      | .some $ .conv .. => .conv
+      | .some $ .convSentinel .. => .conv
     match state.mctx.findDecl? goal with
     | .some mvarDecl =>
-      let serializedGoal ← serializeGoal options goal mvarDecl (parentDecl? := parentDecl?)
-      pure serializedGoal
+      let serializedGoal ← serializeGoal options goal mvarDecl (parentDecl? := .none)
+      pure { serializedGoal with fragment }
     | .none => throwError s!"Metavariable does not exist in context {goal.name}"
 
 /-- Print the metavariables in a readable format -/
@@ -592,6 +608,11 @@ protected def GoalState.diag (goalState: GoalState) (parent?: Option GoalState :
     userNameToString : Name → String
       | .anonymous => ""
       | other => s!"[{other}]"
-    parentHasMVar (mvarId: MVarId): Bool := parent?.map (λ state => state.mctx.decls.contains mvarId) |>.getD true
+    parentHasMVar (mvarId: MVarId): Bool := match parent? with
+      | .some state => state.mctx.decls.contains mvarId
+      | .none => true
+
+initialize
+  registerTraceClass `Pantograph.Delate
 
 end Pantograph

Pantograph/Elab.lean

@@ -6,7 +6,8 @@ namespace Pantograph
 -- Functions for creating contexts and states
 @[export pantograph_default_elab_context]
 def defaultElabContext: Elab.Term.Context := {
-    errToSorry := false
+    declName? := .some `mystery,
+    errToSorry := false,
   }
 
 /-- Read syntax object from string -/

Pantograph/Environment.lean

@@ -4,6 +4,7 @@ import Pantograph.Protocol
 import Pantograph.Serial
 import Lean.Environment
 import Lean.Replay
+import Lean.Util.Path
 
 open Lean
 open Pantograph
@@ -13,7 +14,7 @@ namespace Pantograph.Environment
 @[export pantograph_is_name_internal]
 def isNameInternal (n: Name): Bool :=
   -- Returns true if the name is an implementation detail which should not be shown to the user.
-  n.isAuxLemma ∨ n.hasMacroScopes
+  isAuxLemma n ∨ n.hasMacroScopes
 
 /-- Catalog all the non-internal and safe names -/
 @[export pantograph_environment_catalog]
@@ -24,8 +25,11 @@ def env_catalog (env: Environment): Array Name := env.constants.fold (init := #[
 
 @[export pantograph_environment_module_of_name]
 def module_of_name (env: Environment) (name: Name): Option Name := do
-  let moduleId ←  env.getModuleIdxFor? name
-  return env.allImportedModuleNames.get! moduleId.toNat
+  let moduleId ← env.getModuleIdxFor? name
+  if h : moduleId.toNat < env.allImportedModuleNames.size then
+    return env.allImportedModuleNames[moduleId.toNat]
+  else
+    .none
 
 def toCompactSymbolName (n: Name) (info: ConstantInfo): String :=
   let pref := match info with
@@ -49,12 +53,12 @@ def describe (_: Protocol.EnvDescribe): CoreM Protocol.EnvDescribeResult := do
     imports := env.header.imports.map toString,
     modules := env.header.moduleNames.map (·.toString),
   }
-def moduleRead (args: Protocol.EnvModuleRead): CoreM (Protocol.CR Protocol.EnvModuleReadResult) := do
+def moduleRead (args: Protocol.EnvModuleRead): CoreM Protocol.EnvModuleReadResult := do
   let env ← Lean.MonadEnv.getEnv
   let .some i := env.header.moduleNames.findIdx? (· == args.module.toName) |
-    return .error $ Protocol.errorIndex s!"Module not found {args.module}"
+    throwError s!"Module not found {args.module}"
   let data := env.header.moduleData[i]!
-  return .ok {
+  return {
     imports := data.imports.map toString,
     constNames := data.constNames.map (·.toString),
     extraConstNames := data.extraConstNames.map (·.toString),
@@ -66,15 +70,13 @@ def catalog (_: Protocol.EnvCatalog): CoreM Protocol.EnvCatalogResult := do
     | .some x => acc.push x
     | .none => acc)
   return { symbols := names }
-def inspect (args: Protocol.EnvInspect) (options: @&Protocol.Options): CoreM (Protocol.CR Protocol.EnvInspectResult) := do
+def inspect (args: Protocol.EnvInspect) (options: @&Protocol.Options): Protocol.FallibleT CoreM Protocol.EnvInspectResult := do
   let env ← Lean.MonadEnv.getEnv
   let name :=  args.name.toName
   let info? := env.find? name
-  let info ← match info? with
-    | none => return .error $ Protocol.errorIndex s!"Symbol not found {args.name}"
-    | some info => pure info
+  let .some info := info? | Protocol.throw $ Protocol.errorIndex s!"Symbol not found {args.name}"
   let module? := env.getModuleIdxFor? name >>=
-    (λ idx => env.allImportedModuleNames.get? idx.toNat)
+    (λ idx => env.allImportedModuleNames[idx.toNat]?)
   let value? := match args.value?, info with
     | .some true, _ => info.value?
     | .some false, _ => .none
@@ -88,7 +90,6 @@ def inspect (args: Protocol.EnvInspect) (options: @&Protocol.Options): CoreM (Pr
     isUnsafe := info.isUnsafe,
     value? := ← value?.mapM (λ v => serializeExpression options v |>.run'),
     publicName? := Lean.privateToUserName? name |>.map (·.toString),
-    -- BUG: Warning: getUsedConstants here will not include projections. This is a known bug.
     typeDependency? := if args.dependency?.getD false
       then .some <| type.getUsedConstants.map (λ n => serializeName n)
       else .none,
@@ -130,57 +131,71 @@ def inspect (args: Protocol.EnvInspect) (options: @&Protocol.Options): CoreM (Pr
       } }
     | _ => pure core
   let result ← if args.source?.getD false then
-      let srcSearchPath ← initSrcSearchPath
-      let sourceUri? ← module?.bindM (Server.documentUriFromModule srcSearchPath ·)
-      let declRange? ← findDeclarationRanges? name
-      let sourceStart? := declRange?.map (·.range.pos)
-      let sourceEnd? := declRange?.map (·.range.endPos)
-      .pure {
-        result with
-        sourceUri?,
-        sourceStart?,
-        sourceEnd?,
-      }
+      try
+        let sourceUri? ← module?.bindM (Server.documentUriFromModule? ·)
+        let declRange? ← findDeclarationRanges? name
+        let sourceStart? := declRange?.map (·.range.pos)
+        let sourceEnd? := declRange?.map (·.range.endPos)
+        .pure {
+          result with
+          sourceUri? := sourceUri?.map (toString ·),
+          sourceStart?,
+          sourceEnd?,
+        }
+      catch _e =>
+        .pure result
     else
       .pure result
-  return .ok result
-def addDecl (args: Protocol.EnvAdd): CoreM (Protocol.CR Protocol.EnvAddResult) := do
+  return result
+/-- Elaborates and adds a declaration to the `CoreM` environment. -/
+@[export pantograph_env_add_m]
+def addDecl (name: String) (levels: Array String := #[]) (type?: Option String) (value: String) (isTheorem: Bool)
+    : Protocol.FallibleT CoreM Protocol.EnvAddResult := do
   let env ← Lean.MonadEnv.getEnv
-  let tvM: Elab.TermElabM (Except String (Expr × Expr)) := do
-    let type ← match parseTerm env args.type with
-      | .ok syn => do
-        match ← elabTerm syn with
-        | .error e => return .error e
-        | .ok expr => pure expr
+  let levelParams := levels.toList.map (·.toName)
+  let tvM: Elab.TermElabM (Except String (Expr × Expr)) :=
+    Elab.Term.withLevelNames levelParams do do
+    let expectedType?? : Except String (Option Expr) ← ExceptT.run $ type?.mapM λ type => do
+      match parseTerm env type with
+      | .ok syn => elabTerm syn
+      | .error e => MonadExceptOf.throw e
+    let expectedType? ← match expectedType?? with
+      | .ok t? => pure t?
       | .error e => return .error e
-    let value ← match parseTerm env args.value with
+    let value ← match parseTerm env value with
       | .ok syn => do
         try
-          let expr ← Elab.Term.elabTerm (stx := syn) (expectedType? := .some type)
+          let expr ← Elab.Term.elabTerm (stx := syn) (expectedType? := expectedType?)
           Lean.Elab.Term.synthesizeSyntheticMVarsNoPostponing
           let expr ← instantiateMVars expr
           pure $ expr
         catch ex => return .error (← ex.toMessageData.toString)
       | .error e => return .error e
-    pure $ .ok (type, value)
+    Elab.Term.synthesizeSyntheticMVarsNoPostponing
+    let type ← match expectedType? with
+      | .some t => pure t
+      | .none => Meta.inferType value
+    pure $ .ok (← instantiateMVars type, ← instantiateMVars value)
   let (type, value) ← match ← tvM.run' (ctx := {}) |>.run' with
     | .ok t => pure t
-    | .error e => return .error $ Protocol.errorExpr e
-  let constant := Lean.Declaration.defnDecl <| Lean.mkDefinitionValEx
-    (name := args.name.toName)
-    (levelParams := [])
-    (type := type)
-    (value := value)
-    (hints := Lean.mkReducibilityHintsRegularEx 1)
-    (safety := Lean.DefinitionSafety.safe)
-    (all := [])
-  let env' ← match env.addDecl (← getOptions) constant with
-    | .error e => do
-      let options ← Lean.MonadOptions.getOptions
-      let desc ← (e.toMessageData options).toString
-      return .error $ { error := "kernel", desc }
-    | .ok env' => pure env'
-  Lean.MonadEnv.modifyEnv (λ _ => env')
-  return .ok {}
+    | .error e => Protocol.throw $ Protocol.errorExpr e
+  let decl := if isTheorem then
+    Lean.Declaration.thmDecl <| Lean.mkTheoremValEx
+      (name := name.toName)
+      (levelParams := levelParams)
+      (type := type)
+      (value := value)
+      (all := [])
+  else
+    Lean.Declaration.defnDecl <| Lean.mkDefinitionValEx
+      (name := name.toName)
+      (levelParams := levelParams)
+      (type := type)
+      (value := value)
+      (hints := Lean.mkReducibilityHintsRegularEx 1)
+      (safety := Lean.DefinitionSafety.safe)
+      (all := [])
+  Lean.addDecl decl
+  return {}
 
 end Pantograph.Environment

Pantograph/Frontend/Basic.lean

@@ -6,8 +6,6 @@ open Lean
 namespace Lean.FileMap
 
 /-- Extract the range of a `Syntax` expressed as lines and columns. -/
--- Extracted from the private declaration `Lean.Elab.formatStxRange`,
--- in `Lean.Elab.InfoTree.Main`.
 @[export pantograph_frontend_stx_range]
 protected def stxRange (fileMap : FileMap) (stx : Syntax) : Position × Position :=
   let pos    := stx.getPos?.getD 0
@@ -19,9 +17,10 @@ namespace Lean.PersistentArray
 
 /--
 Drop the first `n` elements of a `PersistentArray`, returning the results as a `List`.
+
+We can't remove the `[Inhabited α]` hypotheses here until
+`PersistentArray`'s `GetElem` instance also does.
 -/
--- We can't remove the `[Inhabited α]` hypotheses here until
--- `PersistentArray`'s `GetElem` instance also does.
 protected def drop [Inhabited α] (t : PersistentArray α) (n : Nat) : List α :=
   List.range (t.size - n) |>.map fun i => t.get! (n + i)
 
@@ -36,10 +35,14 @@ def stxByteRange (stx : Syntax) : String.Pos × String.Pos :=
   let endPos := stx.getTailPos?.getD 0
   (pos, endPos)
 
+structure Context where
+  cancelTk? : Option IO.CancelToken := .none
 
-abbrev FrontendM := Elab.Frontend.FrontendM
+/-- This `FrontendM` comes with more options. -/
+abbrev FrontendM := ReaderT Context Elab.Frontend.FrontendM
 
 structure CompilationStep where
+  scope : Elab.Command.Scope
   fileName : String
   fileMap : FileMap
   src : Substring
@@ -49,14 +52,44 @@ structure CompilationStep where
   msgs : List Message
   trees : List Elab.InfoTree
 
-namespace CompilationStep
+/-- Like `Elab.Frontend.runCommandElabM`, but taking `cancelTk?` into account. -/
+@[inline] def runCommandElabM (x : Elab.Command.CommandElabM α) : FrontendM α := do
+  let config ← read
+  let ctx ← readThe Elab.Frontend.Context
+  let s ← get
+  let cmdCtx : Elab.Command.Context := {
+    cmdPos       := s.cmdPos
+    fileName     := ctx.inputCtx.fileName
+    fileMap      := ctx.inputCtx.fileMap
+    snap?        := none
+    cancelTk?    := config.cancelTk?
+  }
+  match (← liftM <| EIO.toIO' <| (x cmdCtx).run s.commandState) with
+  | Except.error e      => throw <| IO.Error.userError s!"unexpected internal error: {← e.toMessageData.toString}"
+  | Except.ok (a, sNew) => Elab.Frontend.setCommandState sNew; return a
 
-@[export pantograph_frontend_compilation_step_message_strings_m]
-def messageStrings (step: CompilationStep) : IO (Array String) := do
-  List.toArray <$> step.msgs.mapM (·.toString)
-
-end CompilationStep
+def elabCommandAtFrontend (stx : Syntax) : FrontendM Unit := do
+  runCommandElabM do
+    let initMsgs ← modifyGet fun st => (st.messages, { st with messages := {} })
+    Elab.Command.elabCommandTopLevel stx
+    let mut msgs := (← get).messages
+    modify ({ · with messages := initMsgs ++ msgs })
 
+open Elab.Frontend in
+def processCommand : FrontendM Bool := do
+  updateCmdPos
+  let cmdState ← getCommandState
+  let ictx ← getInputContext
+  let pstate ← getParserState
+  let scope := cmdState.scopes.head!
+  let pmctx := { env := cmdState.env, options := scope.opts, currNamespace := scope.currNamespace, openDecls := scope.openDecls }
+  match profileit "parsing" scope.opts fun _ => Parser.parseCommand ictx pmctx pstate cmdState.messages with
+  | (cmd, ps, messages) =>
+    modify fun s => { s with commands := s.commands.push cmd }
+    setParserState ps
+    setMessages messages
+    elabCommandAtFrontend cmd
+    pure (Parser.isTerminalCommand cmd)
 
 /--
 Process one command, returning a `CompilationStep` and
@@ -66,17 +99,19 @@ Process one command, returning a `CompilationStep` and
 def processOneCommand: FrontendM (CompilationStep × Bool) := do
   let s := (← get).commandState
   let before := s.env
-  let done ← Elab.Frontend.processCommand
+  let done ← processCommand
   let stx := (← get).commands.back!
-  let src := (← read).inputCtx.input.toSubstring.extract (← get).cmdPos (← get).parserState.pos
+  let src := (← readThe Elab.Frontend.Context).inputCtx.input.toSubstring.extract
+    (← get).cmdPos
+    (← get).parserState.pos
   let s' := (← get).commandState
   let after := s'.env
   let msgs := s'.messages.toList.drop s.messages.toList.length
   let trees := s'.infoState.trees.drop s.infoState.trees.size
-  let ⟨_, fileName, fileMap⟩  := (← read).inputCtx
-  return ({ fileName, fileMap, src, stx, before, after, msgs, trees }, done)
+  let ⟨_, fileName, fileMap⟩  := (← readThe Elab.Frontend.Context).inputCtx
+  return ({ scope := s.scopes.head!, fileName, fileMap, src, stx, before, after, msgs, trees }, done)
 
-partial def mapCompilationSteps { α } (f: CompilationStep → IO α) : FrontendM (List α) := do
+partial def mapCompilationSteps { α } (f: CompilationStep → FrontendM α) : FrontendM (List α) := do
   let (cmd, done) ← processOneCommand
   if done then
     if cmd.src.isEmpty then
@@ -109,10 +144,11 @@ def createContextStateFromFile
   --let file ← IO.FS.readFile (← findSourcePath module)
   let inputCtx := Parser.mkInputContext file fileName
 
+  let (header, parserState, messages) ← Parser.parseHeader inputCtx
   let (env, parserState, messages) ← match env? with
     | .some env => pure (env, {}, .empty)
     | .none =>
-      let (header, parserState, messages) ← Parser.parseHeader inputCtx
+      -- Only process the header if we don't have an environment.
       let (env, messages) ← Elab.processHeader header opts messages inputCtx
       pure (env, parserState, messages)
   let commandState := Elab.Command.mkState env messages opts

Pantograph/Frontend/Elab.lean

@@ -154,33 +154,31 @@ the draft tactic instead.
 -/
 @[export pantograph_frontend_sorrys_to_goal_state_m]
 def sorrysToGoalState (sorrys : List InfoWithContext) : MetaM AnnotatedGoalState := do
-  let env := sorrys.head? >>= (·.context?) |>.map (·.env) |>.getD (← getEnv)
   assert! !sorrys.isEmpty
-  withEnv env do
-    let goalsM := sorrys.mapM λ i => do
-      match i.info with
-      | .ofTermInfo termInfo  => do
-        let mvarId ← MetaTranslate.translateMVarFromTermInfo termInfo i.context?
+  let goalsM := sorrys.mapM λ i => do
+    match i.info with
+    | .ofTermInfo termInfo  => do
+      let mvarId ← MetaTranslate.translateMVarFromTermInfo termInfo i.context?
+      if (← mvarId.getType).hasSorry then
+        throwError s!"Coupling is not allowed in drafting"
+      return [(mvarId, stxByteRange termInfo.stx)]
+    | .ofTacticInfo tacticInfo => do
+      let mvarIds ← MetaTranslate.translateMVarFromTacticInfoBefore tacticInfo i.context?
+      for mvarId in mvarIds do
         if (← mvarId.getType).hasSorry then
           throwError s!"Coupling is not allowed in drafting"
-        return [(mvarId, stxByteRange termInfo.stx)]
-      | .ofTacticInfo tacticInfo => do
-        let mvarIds ← MetaTranslate.translateMVarFromTacticInfoBefore tacticInfo i.context?
-        for mvarId in mvarIds do
-          if (← mvarId.getType).hasSorry then
-            throwError s!"Coupling is not allowed in drafting"
-        let range := stxByteRange tacticInfo.stx
-        return mvarIds.map (·, range)
-      | _ => panic! "Invalid info"
-    let annotatedGoals := List.flatten (← goalsM.run {} |>.run' {})
-    let goals := annotatedGoals.map Prod.fst
-    let srcBoundaries := annotatedGoals.map Prod.snd
-    let root := match goals with
-      | [] => panic! "No MVars generated"
-      | [g] => g
-      | _ => { name := .anonymous }
-    let state ← GoalState.createFromMVars goals root
-    return { state, srcBoundaries }
+      let range := stxByteRange tacticInfo.stx
+      return mvarIds.map (·, range)
+    | _ => panic! "Invalid info"
+  let annotatedGoals := List.flatten (← goalsM.run {} |>.run' {})
+  let goals := annotatedGoals.map Prod.fst
+  let srcBoundaries := annotatedGoals.map Prod.snd
+  let root := match goals with
+    | [] => panic! "No MVars generated"
+    | [g] => g
+    | _ => { name := .anonymous }
+  let state ← GoalState.createFromMVars goals root
+  return { state, srcBoundaries }
 
 
 @[export pantograph_frontend_collect_new_defined_constants_m]

Pantograph/Frontend/InfoTree.lean

@@ -10,7 +10,7 @@ namespace Lean.Elab
 private def elaboratorToString : Name → String
   | .anonymous => ""
   | n => s!"⟨{n}⟩ "
-private def indent (s : String) : String := "\n".intercalate $ s.splitOn "\n" |>.map ("\t" ++ .)
+private def indent (s : String) : String := "\n".intercalate $ s.splitOn "\n" |>.map ("  " ++ .)
 
 /-- The `Syntax` for a `Lean.Elab.Info`, if there is one. -/
 protected def Info.stx? : Info → Option Syntax
@@ -25,9 +25,9 @@ protected def Info.stx? : Info → Option Syntax
   | .ofCustomInfo         info => info.stx
   | .ofFVarAliasInfo      _    => none
   | .ofFieldRedeclInfo    info => info.stx
-  | .ofOmissionInfo       info => info.stx
   | .ofChoiceInfo         info => info.stx
   | .ofPartialTermInfo    info => info.stx
+  | .ofDelabTermInfo      info => info.stx
 /-- Is the `Syntax` for this `Lean.Elab.Info` original, or synthetic? -/
 protected def Info.isOriginal (i : Info) : Bool :=
   match i.stx? with
@@ -131,20 +131,20 @@ partial def InfoTree.toString (t : InfoTree) (ctx?: Option Elab.ContextInfo := .
   | .node info children =>
     if let some ctx := ctx? then
       let node : String ← match info with
-      | .ofTermInfo    info => pure s!"[term] {(← info.toString ctx)}"
-      | .ofCommandInfo info => pure s!"[command] {(← info.toString ctx)}"
-      | .ofTacticInfo  info => pure s!"[tactic] {(← info.toString ctx)}"
+      | .ofTermInfo    info => pure s!"[term] {info.stx}"
+      | .ofCommandInfo info => pure s!"[command] {info.stx}"
+      | .ofTacticInfo  info => pure s!"[tactic] {info.stx}"
       | .ofMacroExpansionInfo _ => pure "[macro_exp]"
-      | .ofOptionInfo _ => pure "[option]"
+      | .ofOptionInfo info => pure s!"[option] {info.stx}"
       | .ofFieldInfo _ => pure "[field]"
-      | .ofCompletionInfo _ => pure "[completion]"
+      | .ofCompletionInfo info => pure s!"[completion] {info.stx}"
       | .ofUserWidgetInfo _ => pure "[user_widget]"
       | .ofCustomInfo _ => pure "[custom]"
-      | .ofFVarAliasInfo _ => pure "[fvar]"
+      | .ofFVarAliasInfo _ => pure "[fvar_alias]"
       | .ofFieldRedeclInfo _ => pure "[field_redecl]"
-      | .ofOmissionInfo _ => pure "[omission]"
       | .ofChoiceInfo _ => pure "[choice]"
       | .ofPartialTermInfo  _ => pure "[partial_term]"
+      | .ofDelabTermInfo _ => pure "[delab_term]"
       let children := "\n".intercalate (← children.toList.mapM λ t' => do pure $ indent $ ← t'.toString ctx)
       return s!"{node}\n{children}"
     else throw <| IO.userError "No `ContextInfo` available."

Pantograph/Frontend/MetaTranslate.lean

@@ -62,7 +62,7 @@ private partial def translateExpr (srcExpr: Expr) : MetaTranslateM Expr := do
   let sourceMCtx ← getSourceMCtx
   -- We want to create as few mvars as possible
   let (srcExpr, _) := instantiateMVarsCore (mctx := sourceMCtx) srcExpr
-  --IO.println s!"Transform src: {srcExpr}"
+  trace[Pantograph.Frontend.MetaTranslate] "Transform src: {srcExpr}"
   let result ← Core.transform srcExpr λ e => do
     let state ← get
     match e with
@@ -100,10 +100,10 @@ partial def translateLocalDecl (srcLocalDecl: LocalDecl) : MetaTranslateM LocalD
   addTranslatedFVar srcLocalDecl.fvarId fvarId
   match srcLocalDecl with
   | .cdecl index _ userName type bi kind => do
-    --IO.println s!"[CD] {userName} {toString type}"
+    trace[Pantograph.Frontend.MetaTranslate] "[CD] {userName} {toString type}"
     return .cdecl index fvarId userName (← translateExpr type) bi kind
   | .ldecl index _ userName type value nonDep kind => do
-    --IO.println s!"[LD] {toString type} := {toString value}"
+    trace[Pantograph.Frontend.MetaTranslate] "[LD] {toString type} := {toString value}"
     return .ldecl index fvarId userName (← translateExpr type) (← translateExpr value) nonDep kind
 
 partial def translateLCtx : MetaTranslateM LocalContext := do
@@ -111,13 +111,14 @@ partial def translateLCtx : MetaTranslateM LocalContext := do
   let lctx ← MonadLCtx.getLCtx
   assert! lctx.isEmpty
   (← getSourceLCtx).foldlM (λ lctx srcLocalDecl => do
-    let localDecl ← Meta.withLCtx lctx #[] do
+    let localDecl ← Meta.withLCtx' lctx do
       translateLocalDecl srcLocalDecl
     pure $ lctx.addDecl localDecl
   ) lctx
 
 partial def translateMVarId (srcMVarId: MVarId) : MetaTranslateM MVarId := do
   if let .some mvarId' := (← get).mvarMap[srcMVarId]? then
+    trace[Pantograph.Frontend.MetaTranslate] "Existing mvar id {srcMVarId.name} → {mvarId'.name}"
     return mvarId'
   let mvarId' ← Meta.withLCtx .empty #[] do
     let srcDecl := (← getSourceMCtx).findDecl? srcMVarId |>.get!
@@ -134,6 +135,7 @@ partial def translateMVarId (srcMVarId: MVarId) : MetaTranslateM MVarId := do
           let fvars' ← mvarIdPending'.withContext $ fvars.mapM translateExpr
           assignDelayedMVar mvarId' fvars' mvarIdPending'
         pure mvarId'
+  trace[Pantograph.Frontend.MetaTranslate] "Translated {srcMVarId.name} → {mvarId'.name}"
   addTranslatedMVar srcMVarId mvarId'
   return mvarId'
 end
@@ -147,7 +149,9 @@ def translateMVarFromTermInfo (termInfo : Elab.TermInfo) (context? : Option Elab
     let type := termInfo.expectedType?.get!
     let lctx' ← translateLCtx
     let mvar ← Meta.withLCtx lctx' #[] do
+      Meta.withLocalInstances (lctx'.decls.toList.filterMap id) do
       let type' ← translateExpr type
+      trace[Pantograph.Frontend.MetaTranslate] "Translating from term info {← Meta.ppExpr type'}"
       Meta.mkFreshExprSyntheticOpaqueMVar type'
     return mvar.mvarId!
 
@@ -162,4 +166,7 @@ end MetaTranslate
 
 export MetaTranslate (MetaTranslateM)
 
+initialize
+  registerTraceClass `Pantograph.Frontend.MetaTranslate
+
 end Pantograph.Frontend

Pantograph/Goal.lean

@@ -6,28 +6,75 @@ All the functions starting with `try` resume their inner monadic state.
 import Pantograph.Tactic
 import Lean
 
-
 namespace Pantograph
 open Lean
 
+/-- The acting area of a tactic -/
+inductive Site where
+  -- Dormant all other goals
+  | focus (goal : MVarId)
+  -- Move the goal to the first in the list
+  | prefer (goal : MVarId)
+  -- Execute as-is, no goals go dormant
+  | unfocus
+  deriving BEq, Inhabited
+
+instance : Coe MVarId Site where
+  coe := .focus
+instance : ToString Site where
+  toString
+    | .focus { name } => s!"[{name}]"
+    | .prefer { name } => s!"[{name},...]"
+    | .unfocus => "[*]"
+
+/-- Executes a `TacticM` on a site and return affected goals -/
+protected def Site.runTacticM (site : Site)
+  { m } [Monad m] [MonadLiftT Elab.Tactic.TacticM m] [MonadControlT Elab.Tactic.TacticM m] [MonadMCtx m] [MonadError m]
+  (f : m α) : m (α × List MVarId) :=
+  match site with
+  | .focus goal => do
+    Elab.Tactic.setGoals [goal]
+    let a ← f
+    return (a, [goal])
+  | .prefer goal => do
+    let before ← Elab.Tactic.getUnsolvedGoals
+    let otherGoals := before.filter (· != goal)
+    Elab.Tactic.setGoals (goal :: otherGoals)
+    let a ← f
+    let after ← Elab.Tactic.getUnsolvedGoals
+    let parents := before.filter (¬ after.contains ·)
+    Elab.Tactic.pruneSolvedGoals
+    return (a, parents)
+  | .unfocus => do
+    let before ← Elab.Tactic.getUnsolvedGoals
+    let a ← f
+    let after ← Elab.Tactic.getUnsolvedGoals
+    let parents := before.filter (¬ after.contains ·)
+    Elab.Tactic.pruneSolvedGoals
+    return (a, parents)
+
 /--
-Represents an interconnected set of metavariables, or a state in proof search
+Kernel view of the state of a proof
  -/
 structure GoalState where
+  -- Captured `TacticM` state
   savedState : Elab.Tactic.SavedState
 
-  -- The root hole which is the search target
+  -- The root goal which is the search target
   root: MVarId
 
-  -- Parent state metavariable source
-  parentMVar?: Option MVarId
+  /--
+  Parent goals which became assigned or fragmented to produce this state.
+  Note that due to the existence of tactic fragments, parent goals do not
+  necessarily have an expression assignment.
+  -/
+  parentMVars : List MVarId := []
 
-  -- Existence of this field shows that we are currently in `conv` mode.
-  -- (convRhs, goal, dormant)
-  convMVar?: Option (MVarId × MVarId × List MVarId) := .none
-  -- Previous RHS for calc, so we don't have to repeat it every time
-  -- WARNING: If using `state with` outside of `calc`, this must be set to `.none`
-  calcPrevRhs?: Option (MVarId × Expr) := .none
+  -- Any goal associated with a fragment has a partial tactic which has not
+  -- finished executing.
+  fragments : FragmentMap := .empty
+
+def throwNoGoals { m α } [Monad m] [MonadError m] : m α := throwError "no goals to be solved"
 
 @[export pantograph_goal_state_create_m]
 protected def GoalState.create (expr: Expr): Elab.TermElabM GoalState := do
@@ -42,7 +89,6 @@ protected def GoalState.create (expr: Expr): Elab.TermElabM GoalState := do
   return {
     root := root.mvarId!,
     savedState,
-    parentMVar? := .none,
   }
 @[export pantograph_goal_state_create_from_mvars_m]
 protected def GoalState.createFromMVars (goals: List MVarId) (root: MVarId): MetaM GoalState := do
@@ -51,13 +97,19 @@ protected def GoalState.createFromMVars (goals: List MVarId) (root: MVarId): Met
   return {
     root,
     savedState,
-    parentMVar? := .none,
   }
-@[export pantograph_goal_state_is_conv]
-protected def GoalState.isConv (state: GoalState): Bool :=
-  state.convMVar?.isSome
+@[always_inline]
 protected def GoalState.goals (state: GoalState): List MVarId :=
   state.savedState.tactic.goals
+@[always_inline]
+protected def GoalState.mainGoal? (state : GoalState) : Option MVarId :=
+  state.goals.head?
+@[always_inline]
+protected def GoalState.actingGoal? (state : GoalState) (site : Site) : Option MVarId := do
+  match site with
+  | .focus goal | .prefer goal => return goal
+  | .unfocus => state.mainGoal?
+
 @[export pantograph_goal_state_goals]
 protected def GoalState.goalsArray (state: GoalState): Array MVarId := state.goals.toArray
 protected def GoalState.mctx (state: GoalState): MetavarContext :=
@@ -69,80 +121,60 @@ protected def GoalState.env (state: GoalState): Environment :=
 protected def GoalState.metaContextOfGoal (state: GoalState) (mvarId: MVarId): Option Meta.Context := do
   let mvarDecl ← state.mctx.findDecl? mvarId
   return { lctx := mvarDecl.lctx, localInstances := mvarDecl.localInstances }
+@[always_inline]
 protected def GoalState.metaState (state: GoalState): Meta.State :=
   state.savedState.term.meta.meta
+@[always_inline]
 protected def GoalState.coreState (state: GoalState): Core.SavedState :=
   state.savedState.term.meta.core
 
-protected def GoalState.withContext (state: GoalState) (mvarId: MVarId) (m: MetaM α): MetaM α := do
+protected def GoalState.withContext' (state: GoalState) (mvarId: MVarId) (m: MetaM α): MetaM α := do
   mvarId.withContext m |>.run' (← read) state.metaState
-
+protected def GoalState.withContext { m } [MonadControlT MetaM m] [Monad m] (state: GoalState) (mvarId: MVarId) : m α → m α :=
+  Meta.mapMetaM <| state.withContext' mvarId
+/-- Uses context of the first parent -/
 protected def GoalState.withParentContext { n } [MonadControlT MetaM n] [Monad n] (state: GoalState): n α → n α :=
-  Meta.mapMetaM <| state.withContext state.parentMVar?.get!
+  Meta.mapMetaM <| state.withContext' state.parentMVars[0]!
 protected def GoalState.withRootContext { n } [MonadControlT MetaM n] [Monad n] (state: GoalState): n α → n α :=
-  Meta.mapMetaM <| state.withContext state.root
+  Meta.mapMetaM <| state.withContext' state.root
 
-private def GoalState.mvars (state: GoalState): SSet MVarId :=
-  state.mctx.decls.foldl (init := .empty) fun acc k _ => acc.insert k
-protected def GoalState.restoreMetaM (state: GoalState): MetaM Unit :=
+private def restoreCoreMExtra (state : Core.SavedState) : CoreM Unit :=
+  let { nextMacroScope, ngen, .. } := state
+  modifyThe Core.State ({ · with nextMacroScope, ngen })
+-- Restore the name generator and macro scopes of the core state
+protected def GoalState.restoreCoreMExtra (state: GoalState): CoreM Unit :=
+  restoreCoreMExtra state.coreState
+protected def GoalState.restoreMetaM (state: GoalState): MetaM Unit := do
+  state.restoreCoreMExtra
   state.savedState.term.meta.restore
-protected def GoalState.restoreElabM (state: GoalState): Elab.TermElabM Unit :=
+protected def GoalState.restoreElabM (state: GoalState): Elab.TermElabM Unit := do
+  state.restoreCoreMExtra
   state.savedState.term.restore
-private def GoalState.restoreTacticM (state: GoalState) (goal: MVarId): Elab.Tactic.TacticM Unit := do
-  state.savedState.restore
-  Elab.Tactic.setGoals [goal]
 
-@[export pantograph_goal_state_focus]
-protected def GoalState.focus (state: GoalState) (goalId: Nat): Option GoalState := do
-  let goal ← state.savedState.tactic.goals.get? goalId
+/--
+Brings into scope a list of goals. User must ensure `goals` are distinct.
+-/
+@[export pantograph_goal_state_resume]
+protected def GoalState.resume (state : GoalState) (goals : List MVarId) : Except String GoalState := do
+  if ¬ (goals.all (state.mctx.decls.contains ·)) then
+    let invalid_goals := goals.filter (λ goal => ¬ state.mctx.decls.contains goal) |>.map (·.name.toString)
+    .error s!"Goals {invalid_goals} are not in scope"
+  -- Set goals to the goals that have not been assigned yet, similar to the `focus` tactic.
+  let unassigned := goals.filter λ goal =>
+    let isSolved := state.mctx.eAssignment.contains goal || state.mctx.dAssignment.contains goal
+    ¬ isSolved
   return {
     state with
     savedState := {
-      state.savedState with
-      tactic := { goals := [goal] },
-    },
-    calcPrevRhs? := .none,
-  }
-
-/-- Immediately bring all parent goals back into scope. Used in automatic mode -/
-@[export pantograph_goal_state_immediate_resume_parent]
-protected def GoalState.immediateResume (state: GoalState) (parent: GoalState): GoalState :=
-  -- Prune parents solved goals
-  let mctx := state.mctx
-  let parentGoals := parent.goals.filter $ λ goal => mctx.eAssignment.contains goal
-  {
-    state with
-    savedState := {
-      state.savedState with
-      tactic := { goals := state.goals ++ parentGoals },
+      term := state.savedState.term,
+      tactic := { goals := unassigned },
     },
   }
-
-/--
-Brings into scope a list of goals
--/
-@[export pantograph_goal_state_resume]
-protected def GoalState.resume (state: GoalState) (goals: List MVarId): Except String GoalState :=
-  if ¬ (goals.all (λ goal => state.mvars.contains goal)) then
-    let invalid_goals := goals.filter (λ goal => ¬ state.mvars.contains goal) |>.map (·.name.toString)
-    .error s!"Goals {invalid_goals} are not in scope"
-  else
-    -- Set goals to the goals that have not been assigned yet, similar to the `focus` tactic.
-    let unassigned := goals.filter (λ goal =>
-      let mctx := state.mctx
-      ¬(mctx.eAssignment.contains goal || mctx.dAssignment.contains goal))
-    .ok {
-      state with
-      savedState := {
-        term := state.savedState.term,
-        tactic := { goals := unassigned },
-      },
-    }
 /--
 Brings into scope all goals from `branch`
 -/
 @[export pantograph_goal_state_continue]
-protected def GoalState.continue (target: GoalState) (branch: GoalState): Except String GoalState :=
+protected def GoalState.continue (target : GoalState) (branch : GoalState) : Except String GoalState :=
   if !target.goals.isEmpty then
     .error s!"Target state has unresolved goals"
   else if target.root != branch.root then
@@ -151,39 +183,239 @@ protected def GoalState.continue (target: GoalState) (branch: GoalState): Except
     target.resume (goals := branch.goals)
 
 @[export pantograph_goal_state_root_expr]
-protected def GoalState.rootExpr? (goalState: GoalState): Option Expr := do
+protected def GoalState.rootExpr? (goalState : GoalState) : Option Expr := do
   if goalState.root.name == .anonymous then
     .none
   let expr ← goalState.mctx.eAssignment.find? goalState.root
   let (expr, _) := instantiateMVarsCore (mctx := goalState.mctx) (e := expr)
-  if expr.hasExprMVar then
-    -- Must not assert that the goal state is empty here. We could be in a branch goal.
-    --assert! ¬goalState.goals.isEmpty
-    .none
-  else
-    assert! goalState.goals.isEmpty
-    return expr
-@[export pantograph_goal_state_parent_expr]
-protected def GoalState.parentExpr? (goalState: GoalState): Option Expr := do
-  let parent ← goalState.parentMVar?
-  let expr := goalState.mctx.eAssignment.find! parent
-  let (expr, _) := instantiateMVarsCore (mctx := goalState.mctx) (e := expr)
   return expr
+@[export pantograph_goal_state_is_solved]
+protected def GoalState.isSolved (goalState : GoalState) : Bool :=
+  let solvedRoot := match goalState.rootExpr? with
+    | .some e => ¬ e.hasExprMVar
+    | .none => true
+  goalState.goals.isEmpty && solvedRoot
 @[export pantograph_goal_state_get_mvar_e_assignment]
 protected def GoalState.getMVarEAssignment (goalState: GoalState) (mvarId: MVarId): Option Expr := do
   let expr ← goalState.mctx.eAssignment.find? mvarId
   let (expr, _) := instantiateMVarsCore (mctx := goalState.mctx) (e := expr)
   return expr
+@[export pantograph_goal_state_parent_exprs]
+protected def GoalState.parentExprs (state : GoalState) : List (Except Fragment Expr) :=
+  state.parentMVars.map λ goal => match state.getMVarEAssignment goal with
+    | .some e => .ok e
+    -- A parent goal which is not assigned must have a fragment
+    | .none => .error state.fragments[goal]!
+@[always_inline]
+protected def GoalState.hasUniqueParent (state : GoalState) : Bool :=
+  state.parentMVars.length == 1
+@[always_inline]
+protected def GoalState.parentExpr! (state : GoalState) : Expr :=
+  assert! state.parentMVars.length == 1
+  (state.getMVarEAssignment state.parentMVars[0]!).get!
+
+deriving instance BEq for DelayedMetavarAssignment
+
+/-- Given states `dst`, `src`, and `src'`, where `dst` and `src'` are
+descendants of `src`, replay the differential `src' - src` in `dst`. Colliding
+metavariable and lemma names will be automatically renamed to ensure there is no
+collision. This implements branch unification. Unification might be impossible
+if conflicting assignments exist. We also assume the monotonicity property: In a
+chain of descending goal states, a mvar cannot be unassigned, and once assigned
+its assignment cannot change. -/
+@[export pantograph_goal_state_replay_m]
+protected def GoalState.replay (dst : GoalState) (src src' : GoalState) : CoreM GoalState :=
+  withTraceNode `Pantograph.GoalState.replay (fun _ => return m!"replay") do
+  let srcNGen := src.coreState.ngen
+  let srcNGen' := src'.coreState.ngen
+  let dstNGen := dst.coreState.ngen
+  assert! srcNGen.namePrefix == srcNGen'.namePrefix
+  assert! srcNGen.namePrefix == dstNGen.namePrefix
+  assert! src.mctx.depth == src'.mctx.depth
+  assert! src.mctx.depth == dst.mctx.depth
+
+  let diffNGenIdx := dst.coreState.ngen.idx - srcNGen.idx
+
+  let env ← dst.coreState.env.replayConsts src.env src'.env (skipExisting := true)
+
+  trace[Pantograph.GoalState.replay] "Merging ngen {srcNGen.idx} -> ({srcNGen'.idx}, {dstNGen.idx})"
+  -- True if the name is generated after `src`
+  let isNewName : Name → Bool
+    | .num pref n =>
+      pref == srcNGen.namePrefix ∧ n ≥ srcNGen.idx
+    | _ => false
+  let mapId : Name → Name
+    | id@(.num pref n) =>
+      if isNewName id then
+        .num pref (n + diffNGenIdx)
+      else
+        id
+    | id => id
+  let mapMVar : MVarId → MVarId
+    | { name } => ⟨mapId name⟩
+  let rec mapLevel : Level → Level
+    | .succ x => .succ (mapLevel x)
+    | .max l1 l2 => .max (mapLevel l1) (mapLevel l2)
+    | .imax l1 l2 => .imax (mapLevel l1) (mapLevel l2)
+    | .mvar { name } => .mvar ⟨mapId name⟩
+    | l => l
+  let mapExpr (e : Expr) : CoreM Expr := Core.transform e λ
+    | .sort level => pure $ .done $ .sort (mapLevel level)
+    | .mvar { name } => pure $ .done $ .mvar ⟨mapId name⟩
+    | _ => pure .continue
+  let mapDelayedAssignment (d : DelayedMetavarAssignment) : CoreM DelayedMetavarAssignment := do
+    let { mvarIdPending, fvars } := d
+    return {
+      mvarIdPending := mapMVar mvarIdPending,
+      fvars := ← fvars.mapM mapExpr,
+    }
+  let mapLocalDecl (ldecl : LocalDecl) : CoreM LocalDecl := do
+    let ldecl := ldecl.setType (← mapExpr ldecl.type)
+    if let .some value := ldecl.value? then
+      return ldecl.setValue (← mapExpr value)
+    else
+      return ldecl
+
+  let { term := savedTerm@{ meta := savedMeta@{ core, meta := meta@{ mctx, .. } }, .. }, .. } := dst.savedState
+  trace[Pantograph.GoalState.replay] "Merging mvars {src.mctx.mvarCounter} -> ({src'.mctx.mvarCounter}, {dst.mctx.mvarCounter})"
+  let mctx := {
+    mctx with
+    mvarCounter := mctx.mvarCounter + (src'.mctx.mvarCounter - src.mctx.mvarCounter),
+    lDepth := src'.mctx.lDepth.foldl (init := mctx.lDepth) λ acc lmvarId@{ name } depth =>
+      if src.mctx.lDepth.contains lmvarId then
+        acc
+      else
+        acc.insert ⟨mapId name⟩ depth
+    decls := ← src'.mctx.decls.foldlM (init := mctx.decls) λ acc _mvarId@{ name } decl => do
+      if decl.index < src.mctx.mvarCounter then
+        return acc
+      let mvarId := ⟨mapId name⟩
+      let decl := {
+        decl with
+        lctx := ← decl.lctx.foldlM (init := .empty) λ acc decl => do
+          let decl ← mapLocalDecl decl
+          return acc.addDecl decl,
+        type := ← mapExpr decl.type,
+      }
+      return acc.insert mvarId decl
+
+    -- Merge mvar assignments
+    userNames := src'.mctx.userNames.foldl (init := mctx.userNames) λ acc userName mvarId =>
+      if acc.contains userName then
+        acc
+      else
+        acc.insert userName mvarId,
+    lAssignment := src'.mctx.lAssignment.foldl (init := mctx.lAssignment) λ acc lmvarId' l =>
+      let lmvarId := ⟨mapId lmvarId'.name⟩
+      if mctx.lAssignment.contains lmvarId then
+        -- Skip the intersecting assignments for now
+        acc
+      else
+        let l := mapLevel l
+        acc.insert lmvarId l,
+    eAssignment := ← src'.mctx.eAssignment.foldlM (init := mctx.eAssignment) λ acc mvarId' e => do
+      let mvarId := ⟨mapId mvarId'.name⟩
+      if mctx.eAssignment.contains mvarId then
+        -- Skip the intersecting assignments for now
+        return acc
+      else
+        let e ← mapExpr e
+        return acc.insert mvarId e,
+    dAssignment := ← src'.mctx.dAssignment.foldlM (init := mctx.dAssignment) λ acc mvarId' d => do
+      let mvarId := ⟨mapId mvarId'.name⟩
+      if mctx.dAssignment.contains mvarId then
+        return acc
+      else
+        let d ← mapDelayedAssignment d
+        return acc.insert mvarId d
+  }
+  let ngen := {
+    core.ngen with
+    idx := core.ngen.idx + (srcNGen'.idx - srcNGen.idx)
+  }
+  -- Merge conflicting lmvar and mvar assignments using `isDefEq`
+
+  let savedMeta := {
+    savedMeta with
+    core := {
+      core with
+      ngen,
+      env,
+      -- Reset the message log when declaration uses `sorry`
+      messages := {}
+    }
+    meta := {
+      meta with
+      mctx,
+    }
+  }
+  let m : MetaM Meta.SavedState := Meta.withMCtx mctx do
+    restoreCoreMExtra savedMeta.core
+    savedMeta.restore
+
+    for (lmvarId, l') in src'.mctx.lAssignment do
+      if isNewName lmvarId.name then
+        continue
+      let .some l ← getLevelMVarAssignment? lmvarId | continue
+      let l' := mapLevel l'
+      trace[Pantograph.GoalState.replay] "Merging level assignments on {lmvarId.name}"
+      unless ← Meta.isLevelDefEq l l' do
+        throwError "Conflicting assignment of level metavariable {lmvarId.name}"
+    for (mvarId, e') in src'.mctx.eAssignment do
+      if isNewName mvarId.name then
+        continue
+      if ← mvarId.isDelayedAssigned then
+        throwError "Conflicting assignment of expr metavariable (e != d) {mvarId.name}"
+      let .some e ← getExprMVarAssignment? mvarId | continue
+      let e' ← mapExpr e'
+      trace[Pantograph.GoalState.replay] "Merging expr assignments on {mvarId.name}"
+      unless ← Meta.isDefEq e e' do
+        throwError "Conflicting assignment of expr metavariable (e != e) {mvarId.name}"
+    for (mvarId, d') in src'.mctx.dAssignment do
+      if isNewName mvarId.name then
+        continue
+      if ← mvarId.isAssigned then
+        throwError "Conflicting assignment of expr metavariable (d != e) {mvarId.name}"
+      let .some d ← getDelayedMVarAssignment? mvarId | continue
+      trace[Pantograph.GoalState.replay] "Merging expr (delayed) assignments on {mvarId.name}"
+      unless d == d' do
+        throwError "Conflicting assignment of expr metavariable (d != d) {mvarId.name}"
+
+    Meta.saveState
+  let goals := dst.savedState.tactic.goals ++
+    src'.savedState.tactic.goals.map (⟨mapId ·.name⟩)
+  let fragments ← src'.fragments.foldM (init := dst.fragments) λ acc mvarId' fragment' => do
+    let mvarId := ⟨mapId mvarId'.name⟩
+    let fragment ← fragment'.map mapExpr
+    if let .some _fragment0 := acc[mvarId]? then
+      throwError "Conflicting fragments on {mvarId.name}"
+    return acc.insert mvarId fragment
+  return {
+    dst with
+    savedState := {
+      tactic := {
+        goals
+      },
+      term := {
+        savedTerm with
+        meta := ← m.run',
+      },
+    },
+    parentMVars := dst.parentMVars ++ src.parentMVars.map mapMVar,
+    fragments,
+  }
 
 --- Tactic execution functions ---
 
--- Mimics `Elab.Term.logUnassignedUsingErrorInfos`
+/--
+These descendants serve as "seed" mvars. If a MVarError's mvar is related to one
+of these seed mvars, it means an error has occurred when a tactic was executing
+on `src`. `evalTactic`, will not capture these mvars, so we need to manually
+find them and save them into the goal list. See the rationales document for the
+inspiration of this function.
+-/
 private def collectAllErroredMVars (src : MVarId) : Elab.TermElabM (List MVarId) := do
-  -- These descendants serve as "seed" mvars. If a MVarError's mvar is related
-  -- to one of these seed mvars, it means an error has occurred when a tactic
-  -- was executing on `src`. `evalTactic`, will not capture these mvars, so we
-  -- need to manually find them and save them into the goal list.
-
+  -- Mimics `Elab.Term.logUnassignedUsingErrorInfos`
   let descendants ←  Meta.getMVars (.mvar src)
   --let _ ← Elab.Term.logUnassignedUsingErrorInfos descendants
   let mut alreadyVisited : MVarIdSet := {}
@@ -198,6 +430,7 @@ private def collectAllErroredMVars (src : MVarId) : Elab.TermElabM (List MVarId)
         result := mvarDeps.foldl (·.insert ·) result
   return result.toList
 
+/-- Merger of two unique lists -/
 private def mergeMVarLists (li1 li2 : List MVarId) : List MVarId :=
   let li2' := li2.filter (¬ li1.contains ·)
   li1 ++ li2'
@@ -207,96 +440,120 @@ Set `guardMVarErrors` to true to capture mvar errors. Lean will not
 automatically collect mvars from text tactics (vide
 `test_tactic_failure_synthesize_placeholder`)
 -/
-protected def GoalState.step (state: GoalState) (goal: MVarId) (tacticM: Elab.Tactic.TacticM Unit) (guardMVarErrors : Bool := false)
-  : Elab.TermElabM GoalState := do
-  unless (← getMCtx).decls.contains goal do
-    throwError s!"Goal is not in context: {goal.name}"
-  goal.checkNotAssigned `GoalState.step
-  let (_, { goals }) ← tacticM { elaborator := .anonymous } |>.run { goals := [goal] }
+protected def GoalState.step' { α } (state : GoalState) (site : Site) (tacticM : Elab.Tactic.TacticM α) (guardMVarErrors : Bool := false)
+  : Elab.TermElabM (α × GoalState) := do
+  let ((a, parentMVars), { goals }) ← site.runTacticM tacticM
+    |>.run { elaborator := .anonymous }
+    |>.run state.savedState.tactic
   let nextElabState ← MonadBacktrack.saveState
-  Elab.Term.synthesizeSyntheticMVarsNoPostponing
+  --Elab.Term.synthesizeSyntheticMVarsNoPostponing
 
   let goals ← if guardMVarErrors then
-      pure $ mergeMVarLists goals (← collectAllErroredMVars goal)
+      parentMVars.foldlM (init := goals) λ goals parent => do
+        let errors ← collectAllErroredMVars parent
+        return mergeMVarLists goals errors
     else
       pure goals
-  return {
+  let state' := {
     state with
     savedState := { term := nextElabState, tactic := { goals }, },
-    parentMVar? := .some goal,
-    calcPrevRhs? := .none,
+    parentMVars,
   }
+  return (a, state')
+protected def GoalState.step (state : GoalState) (site : Site) (tacticM : Elab.Tactic.TacticM Unit) (guardMVarErrors : Bool := false)
+  : Elab.TermElabM GoalState :=
+  Prod.snd <$> GoalState.step' state site tacticM guardMVarErrors
 
-/-- Response for executing a tactic -/
+/-- Result for executing a tactic, capturing errors in the process -/
 inductive TacticResult where
   -- Goes to next state
-  | success (state: GoalState)
+  | success (state : GoalState) (messages : Array Message)
   -- Tactic failed with messages
-  | failure (messages: Array String)
+  | failure (messages : Array Message)
   -- Could not parse tactic
-  | parseError (message: String)
+  | parseError (message : String)
   -- The given action cannot be executed in the state
-  | invalidAction (message: String)
+  | invalidAction (message : String)
 
-private def dumpMessageLog (prevMessageLength : Nat) : CoreM (Array String) := do
-  let newMessages ← (← Core.getMessageLog).toList.drop prevMessageLength
-    |>.filterMapM λ m => do
-      if m.severity == .error then
-        return .some $ ← m.toString
-      else
-        return .none
+private def dumpMessageLog (prevMessageLength : Nat := 0) : CoreM (List Message) := do
+  let newMessages := (← Core.getMessageLog).toList.drop prevMessageLength
   Core.resetMessageLog
-  return newMessages.toArray
+  return newMessages
+
+/-- Execute a `TermElabM` producing a goal state, capturing the error and turn it into a `TacticResult` -/
+def withCapturingError (elabM : Elab.Term.TermElabM GoalState) : Elab.TermElabM TacticResult := do
+  let messageLog ← Core.getMessageLog
+  unless messageLog.toList.isEmpty do
+    IO.eprintln s!"{← messageLog.toList.mapM (·.toString)}"
+    throwError "Message log must be empty at the beginning."
+  try
+    let state ← elabM
+
+    -- Check if error messages have been generated in the core.
+    let newMessages ← dumpMessageLog
+    let hasErrors := newMessages.any (·.severity == .error)
+    if hasErrors then
+      return .failure newMessages.toArray
+    else
+      return .success state newMessages.toArray
+  catch exception =>
+    let messages ← dumpMessageLog
+    let message := {
+      fileName := ← getFileName,
+      pos := ← getRefPosition,
+      data := exception.toMessageData,
+    }
+    return .failure (message :: messages).toArray
 
 /-- Executes a `TacticM` monad on this `GoalState`, collecting the errors as necessary -/
 protected def GoalState.tryTacticM
-    (state: GoalState) (goal: MVarId) (tacticM: Elab.Tactic.TacticM Unit)
+    (state: GoalState) (site : Site)
+    (tacticM: Elab.Tactic.TacticM Unit)
     (guardMVarErrors : Bool := false)
     : Elab.TermElabM TacticResult := do
-  let prevMessageLength := state.coreState.messages.toList.length
-  try
-    let nextState ← state.step goal tacticM guardMVarErrors
-
-    -- Check if error messages have been generated in the core.
-    let newMessages ← dumpMessageLog prevMessageLength
-    if ¬ newMessages.isEmpty then
-      return .failure newMessages
-    return .success nextState
-  catch exception =>
-    match exception with
-    | .internal _ => return .failure $ ← dumpMessageLog prevMessageLength
-    | _ => return .failure #[← exception.toMessageData.toString]
+  state.restoreElabM
+  withCapturingError do
+    state.step site tacticM guardMVarErrors
 
 /-- Execute a string tactic on given state. Restores TermElabM -/
 @[export pantograph_goal_state_try_tactic_m]
-protected def GoalState.tryTactic (state: GoalState) (goal: MVarId) (tactic: String):
+protected def GoalState.tryTactic (state: GoalState) (site : Site) (tactic: String):
     Elab.TermElabM TacticResult := do
   state.restoreElabM
+  let .some goal := state.actingGoal? site | throwNoGoals
+  if let .some fragment := state.fragments[goal]? then
+    return ← withCapturingError do
+      let (fragments, state') ← state.step' site do
+        fragment.step goal tactic $ state.fragments.erase goal
+      return { state' with fragments }
+  -- Normal tactic without fragment
   let tactic ← match Parser.runParserCategory
-    (env := ← MonadEnv.getEnv)
-    (catName := if state.isConv then `conv else `tactic)
+    (env := ← getEnv)
+    (catName := `tactic)
     (input := tactic)
     (fileName := ← getFileName) with
     | .ok stx => pure $ stx
     | .error error => return .parseError error
-  state.tryTacticM goal (Elab.Tactic.evalTactic tactic) true
+  let tacticM := Elab.Tactic.evalTactic tactic
+  withCapturingError do
+    state.step site tacticM (guardMVarErrors := true)
 
-protected def GoalState.tryAssign (state: GoalState) (goal: MVarId) (expr: String):
-      Elab.TermElabM TacticResult := do
+-- Specialized Tactics
+
+protected def GoalState.tryAssign (state : GoalState) (site : Site) (expr : String)
+    : Elab.TermElabM TacticResult := do
   state.restoreElabM
   let expr ← match Parser.runParserCategory
-    (env := ← MonadEnv.getEnv)
+    (env := ← getEnv)
     (catName := `term)
     (input := expr)
     (fileName := ← getFileName) with
     | .ok syn => pure syn
     | .error error => return .parseError error
-  state.tryTacticM goal $ Tactic.evalAssign expr
+  state.tryTacticM site $ Tactic.evalAssign expr
 
--- Specialized Tactics
-
-protected def GoalState.tryLet (state: GoalState) (goal: MVarId) (binderName: String) (type: String):
-      Elab.TermElabM TacticResult := do
+protected def GoalState.tryLet (state : GoalState) (site : Site) (binderName : String) (type : String)
+    : Elab.TermElabM TacticResult := do
   state.restoreElabM
   let type ← match Parser.runParserCategory
     (env := ← MonadEnv.getEnv)
@@ -305,150 +562,59 @@ protected def GoalState.tryLet (state: GoalState) (goal: MVarId) (binderName: St
     (fileName := ← getFileName) with
     | .ok syn => pure syn
     | .error error => return .parseError error
-  state.tryTacticM goal $ Tactic.evalLet binderName.toName type
+  state.tryTacticM site $ Tactic.evalLet binderName.toName type
 
 /-- Enter conv tactic mode -/
-protected def GoalState.conv (state: GoalState) (goal: MVarId):
+@[export pantograph_goal_state_conv_enter_m]
+protected def GoalState.convEnter (state : GoalState) (site : Site) :
       Elab.TermElabM TacticResult := do
-  if state.convMVar?.isSome then
+  let .some goal := state.actingGoal? site | throwNoGoals
+  if let .some (.conv ..) := state.fragments[goal]? then
     return .invalidAction "Already in conv state"
-  goal.checkNotAssigned `GoalState.conv
-  let tacticM :  Elab.Tactic.TacticM (Elab.Tactic.SavedState × MVarId) := do
-    state.restoreTacticM goal
-
-    -- See Lean.Elab.Tactic.Conv.convTarget
-    let convMVar ← Elab.Tactic.withMainContext do
-      let (rhs, newGoal) ← Elab.Tactic.Conv.mkConvGoalFor (← Elab.Tactic.getMainTarget)
-      Elab.Tactic.replaceMainGoal [newGoal.mvarId!]
-      pure rhs.mvarId!
-    return (← MonadBacktrack.saveState, convMVar)
-  try
-    let (nextSavedState, convRhs) ← tacticM { elaborator := .anonymous } |>.run' state.savedState.tactic
-    -- Other goals are now dormant
-    let otherGoals := state.goals.filter $ λ g => g != goal
-    return .success {
-      root := state.root,
-      savedState := nextSavedState
-      parentMVar? := .some goal,
-      convMVar? := .some (convRhs, goal, otherGoals),
-      calcPrevRhs? := .none
-    }
-  catch exception =>
-    return .failure #[← exception.toMessageData.toString]
-
-/-- Exit from `conv` mode. Resumes all goals before the mode starts and applys the conv -/
-@[export pantograph_goal_state_conv_exit_m]
-protected def GoalState.convExit (state: GoalState):
-      Elab.TermElabM TacticResult := do
-  let (convRhs, convGoal, _) ← match state.convMVar? with
-    | .some mvar => pure mvar
-    | .none => return .invalidAction "Not in conv state"
-  let tacticM :  Elab.Tactic.TacticM Elab.Tactic.SavedState:= do
-    -- Vide `Lean.Elab.Tactic.Conv.convert`
-    state.savedState.restore
-
-    -- Close all existing goals with `refl`
-    for mvarId in (← Elab.Tactic.getGoals) do
-      liftM <| mvarId.refl <|> mvarId.inferInstance <|> pure ()
-    Elab.Tactic.pruneSolvedGoals
-    unless (← Elab.Tactic.getGoals).isEmpty do
-      throwError "convert tactic failed, there are unsolved goals\n{Elab.goalsToMessageData (← Elab.Tactic.getGoals)}"
-
-    Elab.Tactic.setGoals [convGoal]
-
-    let targetNew ← instantiateMVars (.mvar convRhs)
-    let proof ← instantiateMVars (.mvar convGoal)
-
-    Elab.Tactic.liftMetaTactic1 fun mvarId => mvarId.replaceTargetEq targetNew proof
-    MonadBacktrack.saveState
-  try
-    let nextSavedState ← tacticM { elaborator := .anonymous } |>.run' state.savedState.tactic
-    return .success {
-      root := state.root,
-      savedState := nextSavedState
-      parentMVar? := .some convGoal,
-      convMVar? := .none
-      calcPrevRhs? := .none
-    }
-  catch exception =>
-    return .failure #[← exception.toMessageData.toString]
-
-protected def GoalState.calcPrevRhsOf? (state: GoalState) (goal: MVarId): Option Expr := do
-  let (mvarId, rhs) ← state.calcPrevRhs?
-  if mvarId == goal then
-    .some rhs
-  else
-    .none
-@[export pantograph_goal_state_try_calc_m]
-protected def GoalState.tryCalc (state: GoalState) (goal: MVarId) (pred: String):
-      Elab.TermElabM TacticResult := do
   state.restoreElabM
-  if state.convMVar?.isSome then
-    return .invalidAction "Cannot initiate `calc` while in `conv` state"
-  let `(term|$pred) ← match Parser.runParserCategory
-    (env := state.env)
-    (catName := `term)
-    (input := pred)
-    (fileName := ← getFileName) with
-    | .ok syn => pure syn
-    | .error error => return .parseError error
-  goal.checkNotAssigned `GoalState.tryCalc
-  let calcPrevRhs? := state.calcPrevRhsOf? goal
-  let decl ← goal.getDecl
-  let target ← instantiateMVars decl.type
-  let tag := decl.userName
-  try
-    goal.withContext do
-
-    let mut step ← Elab.Term.elabType <| ← do
-      if let some prevRhs := calcPrevRhs? then
-        Elab.Term.annotateFirstHoleWithType pred (← Meta.inferType prevRhs)
-      else
-        pure pred
-
-    let some (_, lhs, rhs) ← Elab.Term.getCalcRelation? step |
-      throwErrorAt pred "invalid 'calc' step, relation expected{indentExpr step}"
-    if let some prevRhs := calcPrevRhs? then
-      unless ← Meta.isDefEqGuarded lhs prevRhs do
-        throwErrorAt pred "invalid 'calc' step, left-hand-side is{indentD m!"{lhs} : {← Meta.inferType lhs}"}\nprevious right-hand-side is{indentD m!"{prevRhs} : {← Meta.inferType prevRhs}"}"
-
-    -- Creates a mvar to represent the proof that the calc tactic solves the
-    -- current branch
-    -- In the Lean `calc` tactic this is gobbled up by
-    -- `withCollectingNewGoalsFrom`
-    let mut proof ← Meta.mkFreshExprMVarAt (← getLCtx) (← Meta.getLocalInstances) step
-      (userName := tag ++ `calc)
-    let mvarBranch := proof.mvarId!
-
-    let mut proofType ← Meta.inferType proof
-    let mut remainder? := Option.none
-
-    -- The calc tactic either solves the main goal or leaves another relation.
-    -- Replace the main goal, and save the new goal if necessary
-    unless ← Meta.isDefEq proofType target do
-      let rec throwFailed :=
-        throwError "'calc' tactic failed, has type{indentExpr proofType}\nbut it is expected to have type{indentExpr target}"
-      let some (_, _, rhs) ← Elab.Term.getCalcRelation? proofType | throwFailed
-      let some (r, _, rhs') ← Elab.Term.getCalcRelation? target | throwFailed
-      let lastStep := mkApp2 r rhs rhs'
-      let lastStepGoal ← Meta.mkFreshExprSyntheticOpaqueMVar lastStep tag
-      (proof, proofType) ← Elab.Term.mkCalcTrans proof proofType lastStepGoal lastStep
-      unless ← Meta.isDefEq proofType target do throwFailed
-      remainder? := .some lastStepGoal.mvarId!
-    goal.assign proof
-
-    let goals := [ mvarBranch ] ++ remainder?.toList
-    let calcPrevRhs? := remainder?.map $ λ g => (g, rhs)
-    return .success {
-      root := state.root,
-      savedState := {
-        term := ← MonadBacktrack.saveState,
-        tactic := { goals },
-      },
-      parentMVar? := .some goal,
-      calcPrevRhs?
+  withCapturingError do
+    let (fragments, state') ← state.step' site Fragment.enterConv
+    return {
+      state' with
+      fragments := fragments.fold (init := state'.fragments) λ acc goal fragment =>
+        acc.insert goal fragment
     }
-  catch exception =>
-    return .failure #[← exception.toMessageData.toString]
+
+/-- Exit from a tactic fragment. -/
+@[export pantograph_goal_state_fragment_exit_m]
+protected def GoalState.fragmentExit (state : GoalState) (site : Site):
+      Elab.TermElabM TacticResult := do
+  let .some goal := state.actingGoal? site | throwNoGoals
+  let .some fragment := state.fragments[goal]? |
+    return .invalidAction "Goal does not have a fragment"
+  state.restoreElabM
+  withCapturingError do
+    let (fragments, state') ← state.step' goal (fragment.exit goal state.fragments)
+    return {
+      state' with
+      fragments,
+    }
+
+protected def GoalState.calcPrevRhsOf? (state : GoalState) (goal : MVarId) : Option Expr := do
+  let .some (.calc prevRhs?) := state.fragments[goal]? | .none
+  prevRhs?
+
+@[export pantograph_goal_state_calc_enter_m]
+protected def GoalState.calcEnter (state : GoalState) (site : Site)
+  : Elab.TermElabM TacticResult := do
+  let .some goal := state.actingGoal? site | throwNoGoals
+  if let .some _ := state.fragments[goal]? then
+    return .invalidAction "Goal already has a fragment"
+  state.restoreElabM
+  withCapturingError do
+    let fragment := Fragment.enterCalc
+    let fragments := state.fragments.insert goal fragment
+    return {
+      state with
+      fragments,
+    }
+
+initialize
+  registerTraceClass `Pantograph.GoalState.replay
 
 end Pantograph

Pantograph/Library.lean

@@ -2,7 +2,7 @@ import Pantograph.Environment
 import Pantograph.Goal
 import Pantograph.Protocol
 import Pantograph.Delate
-import Pantograph.Version
+
 import Lean
 
 namespace Lean
@@ -40,8 +40,6 @@ namespace Pantograph
 
 def runMetaM { α } (metaM: MetaM α): CoreM α :=
   metaM.run'
-def runTermElabM { α } (termElabM: Elab.TermElabM α): CoreM α :=
-  termElabM.run' (ctx := defaultElabContext) |>.run'
 
 def errorI (type desc: String): Protocol.InteractionError := { error := type, desc := desc }
 
@@ -70,86 +68,72 @@ def createCoreContext (options: Array String): IO Core.Context := do
 @[export pantograph_create_core_state]
 def createCoreState (imports: Array String): IO Core.State := do
   let env ← Lean.importModules
-    (imports := imports.map (λ str => { module := str.toName, runtimeOnly := false }))
+    (imports := imports.map (λ str => { module := str.toName }))
     (opts := {})
     (trustLevel := 1)
+    (loadExts := true)
   return { env := env }
 
-@[export pantograph_env_add_m]
-def envAdd (name: String) (type: String) (value: String) (isTheorem: Bool):
-    CoreM (Protocol.CR Protocol.EnvAddResult) :=
-  Environment.addDecl { name, type, value, isTheorem }
-
 @[export pantograph_parse_elab_type_m]
-def parseElabType (type: String): Elab.TermElabM (Protocol.CR Expr) := do
+def parseElabType (type: String): Protocol.FallibleT Elab.TermElabM Expr := do
   let env ← MonadEnv.getEnv
   let syn ← match parseTerm env type with
-    | .error str => return .error $ errorI "parsing" str
+    | .error str => Protocol.throw $ errorI "parsing" str
     | .ok syn => pure syn
   match ← elabType syn with
-  | .error str => return .error $ errorI "elab" str
-  | .ok expr => return .ok (← instantiateMVars expr)
+  | .error str => Protocol.throw $ errorI "elab" str
+  | .ok expr => return (← instantiateMVars expr)
 
 /-- This must be a TermElabM since the parsed expr contains extra information -/
 @[export pantograph_parse_elab_expr_m]
-def parseElabExpr (expr: String) (expectedType?: Option String := .none): Elab.TermElabM (Protocol.CR Expr) := do
+def parseElabExpr (expr: String) (expectedType?: Option String := .none): Protocol.FallibleT Elab.TermElabM Expr := do
   let env ← MonadEnv.getEnv
-  let expectedType? ← match ← expectedType?.mapM parseElabType with
-    | .none => pure $ .none
-    | .some (.ok t) => pure $ .some t
-    | .some (.error e) => return .error e
+  let expectedType? ← expectedType?.mapM parseElabType
   let syn ← match parseTerm env expr with
-    | .error str => return .error $ errorI "parsing" str
+    | .error str => Protocol.throw $ errorI "parsing" str
     | .ok syn => pure syn
   match ← elabTerm syn expectedType? with
-  | .error str => return .error $ errorI "elab" str
-  | .ok expr => return .ok (← instantiateMVars expr)
+  | .error str => Protocol.throw $ errorI "elab" str
+  | .ok expr => return (← instantiateMVars expr)
 
 @[export pantograph_expr_echo_m]
-def exprEcho (expr: String) (expectedType?: Option String := .none) (levels: Array String := #[])  (options: @&Protocol.Options := {}):
-    CoreM (Protocol.CR Protocol.ExprEchoResult) :=
-  runTermElabM $ Elab.Term.withLevelNames (levels.toList.map (·.toName)) do
-    let expr ← match ← parseElabExpr expr expectedType? with
-      | .error e => return .error e
-      | .ok expr => pure expr
-    try
-      let type ← unfoldAuxLemmas (← Meta.inferType expr)
-      return .ok {
-          type := (← serializeExpression options type),
-          expr := (← serializeExpression options expr)
-      }
-    catch exception =>
-      return .error $ errorI "typing" (← exception.toMessageData.toString)
+def exprEcho (expr: String) (expectedType?: Option String := .none) (options: @&Protocol.Options := {}):
+    Protocol.FallibleT Elab.TermElabM Protocol.ExprEchoResult := do
+  let expr ← parseElabExpr expr expectedType?
+  try
+    let type ← unfoldAuxLemmas (← Meta.inferType expr)
+    return {
+        type := (← serializeExpression options type),
+        expr := (← serializeExpression options expr),
+    }
+  catch exception =>
+    Protocol.throw $ errorI "typing" (← exception.toMessageData.toString)
 
 @[export pantograph_goal_start_expr_m]
-def goalStartExpr (expr: String) (levels: Array String): CoreM (Protocol.CR GoalState) :=
-  runTermElabM $ Elab.Term.withLevelNames (levels.toList.map (·.toName)) do
-    let expr ← match ← parseElabType expr with
-      | .error e => return .error e
-      | .ok expr => pure $ expr
-    return .ok $ ← GoalState.create expr
-
-@[export pantograph_goal_resume]
-def goalResume (target: GoalState) (goals: Array String): Except String GoalState :=
-  target.resume (goals.map (λ n => { name := n.toName }) |>.toList)
+def goalStartExpr (expr: String) : Protocol.FallibleT Elab.TermElabM GoalState := do
+  let t ← parseElabType expr
+  GoalState.create t
 
 @[export pantograph_goal_serialize_m]
 def goalSerialize (state: GoalState) (options: @&Protocol.Options): CoreM (Array Protocol.Goal) :=
-  runMetaM <| state.serializeGoals (parent := .none) options
+  runMetaM <| state.serializeGoals options
 
 @[export pantograph_goal_print_m]
-def goalPrint (state: GoalState) (rootExpr: Bool) (parentExpr: Bool) (goals: Bool) (extraMVars : Array String) (options: @&Protocol.Options)
+def goalPrint (state: GoalState) (rootExpr: Bool) (parentExprs: Bool) (goals: Bool) (extraMVars : Array String) (options: @&Protocol.Options)
   : CoreM Protocol.GoalPrintResult := runMetaM do
   state.restoreMetaM
 
+  let rootExpr? := state.rootExpr?
   let root? ← if rootExpr then
-      state.rootExpr?.mapM λ expr => state.withRootContext do
+      rootExpr?.mapM λ expr => state.withRootContext do
         serializeExpression options (← instantiateAll expr)
     else
       pure .none
-  let parent? ← if parentExpr then
-      state.parentExpr?.mapM λ expr => state.withParentContext do
-        serializeExpression options (← instantiateAll expr)
+  let parentExprs? ← if parentExprs then
+      .some <$> state.parentMVars.mapM λ parent => parent.withContext do
+        let val? := state.getMVarEAssignment parent
+        val?.mapM λ val => do
+          serializeExpression options (← instantiateAll val)
     else
       pure .none
   let goals ← if goals then
@@ -162,70 +146,50 @@ def goalPrint (state: GoalState) (rootExpr: Bool) (parentExpr: Bool) (goals: Boo
     state.withContext mvarId do
       let .some expr ← getExprMVarAssignment? mvarId | return {}
       serializeExpression options (← instantiateAll expr)
+  let env ← getEnv
   return {
     root?,
-    parent?,
+    parentExprs?,
     goals,
     extraMVars,
+    rootHasSorry := rootExpr?.map (·.hasSorry) |>.getD false,
+    rootHasUnsafe := rootExpr?.map (env.hasUnsafe ·) |>.getD false,
+    rootHasMVar := rootExpr?.map (·.hasExprMVar) |>.getD false,
   }
 
-@[export pantograph_goal_tactic_m]
-def goalTactic (state: GoalState) (goal:  MVarId) (tactic: String): CoreM TacticResult :=
-  runTermElabM <| state.tryTactic goal tactic
-@[export pantograph_goal_assign_m]
-def goalAssign (state: GoalState) (goal: MVarId) (expr: String): CoreM TacticResult :=
-  runTermElabM <| state.tryAssign goal expr
 @[export pantograph_goal_have_m]
-protected def GoalState.tryHave (state: GoalState) (goal: MVarId) (binderName: String) (type: String): CoreM TacticResult := do
+protected def GoalState.tryHave (state: GoalState) (site : Site) (binderName: String) (type: String): Elab.TermElabM TacticResult := do
   let type ← match (← parseTermM type) with
     | .ok syn => pure syn
     | .error error => return .parseError error
-  runTermElabM do
-    state.restoreElabM
-    state.tryTacticM goal $ Tactic.evalHave binderName.toName type
+  state.restoreElabM
+  state.tryTacticM site $ Tactic.evalHave binderName.toName type
 @[export pantograph_goal_try_define_m]
-protected def GoalState.tryDefine (state: GoalState) (goal: MVarId) (binderName: String) (expr: String): CoreM TacticResult := do
+protected def GoalState.tryDefine (state: GoalState) (site : Site) (binderName: String) (expr: String): Elab.TermElabM TacticResult := do
   let expr ← match (← parseTermM expr) with
     | .ok syn => pure syn
     | .error error => return .parseError error
-  runTermElabM do
-    state.restoreElabM
-    state.tryTacticM goal (Tactic.evalDefine binderName.toName expr)
-@[export pantograph_goal_try_motivated_apply_m]
-protected def GoalState.tryMotivatedApply (state: GoalState) (goal: MVarId) (recursor: String):
-      Elab.TermElabM TacticResult := do
   state.restoreElabM
-  let recursor ← match (← parseTermM recursor) with
-    | .ok syn => pure syn
-    | .error error => return .parseError error
-  state.tryTacticM goal (tacticM := Tactic.evalMotivatedApply recursor)
-@[export pantograph_goal_try_no_confuse_m]
-protected def GoalState.tryNoConfuse (state: GoalState) (goal: MVarId) (eq: String):
-      Elab.TermElabM TacticResult := do
-  state.restoreElabM
-  let eq ← match (← parseTermM eq) with
-    | .ok syn => pure syn
-    | .error error => return .parseError error
-  state.tryTacticM goal (tacticM := Tactic.evalNoConfuse eq)
+  state.tryTacticM site $ Tactic.evalDefine binderName.toName expr
 @[export pantograph_goal_try_draft_m]
-protected def GoalState.tryDraft (state: GoalState) (goal: MVarId) (expr: String): CoreM TacticResult := do
+protected def GoalState.tryDraft (state: GoalState) (site : Site) (expr: String): Elab.TermElabM TacticResult := do
   let expr ← match (← parseTermM expr) with
     | .ok syn => pure syn
     | .error error => return .parseError error
-  runTermElabM do
-    state.restoreElabM
-    state.tryTacticM goal (Tactic.evalDraft expr)
-@[export pantograph_goal_let_m]
-def goalLet (state: GoalState) (goal: MVarId) (binderName: String) (type: String): CoreM TacticResult :=
-  runTermElabM <| state.tryLet goal binderName type
-@[export pantograph_goal_conv_m]
-def goalConv (state: GoalState) (goal: MVarId): CoreM TacticResult :=
-  runTermElabM <| state.conv goal
-@[export pantograph_goal_conv_exit_m]
-def goalConvExit (state: GoalState): CoreM TacticResult :=
-  runTermElabM <| state.convExit
-@[export pantograph_goal_calc_m]
-def goalCalc (state: GoalState) (goal: MVarId) (pred: String): CoreM TacticResult :=
-  runTermElabM <| state.tryCalc goal pred
+  state.restoreElabM
+  state.tryTacticM site $ Tactic.evalDraft expr
+
+-- Cancel the token after a timeout.
+@[export pantograph_run_cancel_token_with_timeout_m]
+def runCancelTokenWithTimeout (cancelToken : IO.CancelToken) (timeout : UInt32) : IO Unit := do
+  let _ ← EIO.asTask do
+    IO.sleep timeout
+    cancelToken.set
+  return ()
+
+def spawnCancelToken (timeout : UInt32) : IO IO.CancelToken := do
+  let token ← IO.CancelToken.new
+  runCancelTokenWithTimeout token timeout
+  return token
 
 end Pantograph

Pantograph/Protocol.lean

@@ -6,6 +6,7 @@ its field names to avoid confusion with error messages generated by the REPL.
 -/
 import Lean.Data.Json
 import Lean.Data.Position
+import Lean.Message
 
 namespace Pantograph.Protocol
 
@@ -30,6 +31,8 @@ structure Options where
   printImplementationDetailHyps: Bool := false
   -- If this is set to `true`, goals will never go dormant, so you don't have to manage resumption
   automaticMode: Bool := true
+  -- Timeout for tactics and operations that could potentially execute a tactic
+  timeout: Nat := 0
   deriving Lean.ToJson
 
 abbrev OptionsT := ReaderT Options
@@ -58,20 +61,23 @@ structure Variable where
   type?: Option Expression  := .none
   value?: Option Expression := .none
   deriving Lean.ToJson
+inductive Fragment where
+  | tactic
+  | conv
+  | calc
+  deriving BEq, DecidableEq, Repr, Lean.ToJson
 structure Goal where
-  name: String := ""
   /-- Name of the metavariable -/
-  userName?: Option String  := .none
-  /-- Is the goal in conversion mode -/
-  isConversion: Bool        := false
+  name : String := ""
+  /-- User-facing name -/
+  userName? : Option String  := .none
+  fragment : Fragment := .tactic
   /-- target expression type -/
-  target: Expression
+  target : Expression
   /-- Variables -/
-  vars: Array Variable      := #[]
+  vars : Array Variable      := #[]
   deriving Lean.ToJson
 
-
-
 --- Individual Commands and return types ---
 
 structure Command where
@@ -85,12 +91,10 @@ structure InteractionError where
   deriving Lean.ToJson
 
 def errorIndex (desc: String): InteractionError := { error := "index", desc }
+def errorOperation (desc: String): InteractionError := { error := "operation", desc }
 def errorExpr (desc: String): InteractionError := { error := "expr", desc }
 
 
---- Individual command and return types ---
-
-
 structure Reset where
   deriving Lean.FromJson
 structure Stat where
@@ -103,9 +107,9 @@ structure StatResult where
 -- Return the type of an expression
 structure ExprEcho where
   expr: String
-  type?: Option String
+  type?: Option String := .none
   -- universe levels
-  levels: Option (Array String) := .none
+  levels?: Option (Array String) := .none
   deriving Lean.FromJson
 structure ExprEchoResult where
   expr: Expression
@@ -202,9 +206,10 @@ structure EnvInspectResult where
 
 structure EnvAdd where
   name: String
-  type: String
+  levels?: Option (Array String) := .none
+  type?: Option String := .none
   value: String
-  isTheorem: Bool
+  isTheorem: Bool := false
   deriving Lean.FromJson
 structure EnvAddResult where
   deriving Lean.ToJson
@@ -217,14 +222,15 @@ structure EnvSaveLoadResult where
 
 /-- Set options; See `Options` struct above for meanings -/
 structure OptionsSet where
-  printJsonPretty?: Option Bool
-  printExprPretty?: Option Bool
-  printExprAST?: Option Bool
-  printDependentMVars?: Option Bool
-  noRepeat?: Option Bool
-  printAuxDecls?: Option Bool
-  printImplementationDetailHyps?: Option Bool
-  automaticMode?: Option Bool
+  printJsonPretty?: Option Bool := .none
+  printExprPretty?: Option Bool := .none
+  printExprAST?: Option Bool := .none
+  printDependentMVars?: Option Bool := .none
+  noRepeat?: Option Bool := .none
+  printAuxDecls?: Option Bool := .none
+  printImplementationDetailHyps?: Option Bool := .none
+  automaticMode?: Option Bool := .none
+  timeout?: Option Nat := .none
   deriving Lean.FromJson
 structure OptionsSetResult where
   deriving Lean.ToJson
@@ -235,8 +241,8 @@ structure GoalStart where
   -- Only one of the fields below may be populated.
   expr: Option String     -- Directly parse in an expression
   -- universe levels
-  levels: Option (Array String) := .none
-  copyFrom: Option String -- Copy the type from a theorem in the environment
+  levels?: Option (Array String) := .none
+  copyFrom: Option String := .none -- Copy the type from a theorem in the environment
   deriving Lean.FromJson
 structure GoalStartResult where
   stateId: Nat := 0
@@ -244,34 +250,40 @@ structure GoalStartResult where
   root: String
   deriving Lean.ToJson
 structure GoalTactic where
-  -- Identifiers for tree, state, and goal
   stateId: Nat
-  goalId: Nat := 0
+  -- If omitted, act on the first goal
+  goalId?: Option Nat := .none
+  -- If set to true, goal will not go dormant. Defaults to `automaticMode`
+  autoResume?: Option Bool := .none
   -- One of the fields here must be filled
   tactic?: Option String := .none
+  -- Changes the current category to {"tactic", "calc", "conv"}
+  mode?: Option String := .none
+  -- Assigns an expression to the current goal
   expr?: Option String := .none
   have?: Option String := .none
   let?: Option String := .none
-  calc?: Option String := .none
-  -- true to enter `conv`, `false` to exit. In case of exit the `goalId` is ignored.
-  conv?: Option Bool := .none
   draft?: Option String := .none
 
-  -- In case of the `have` tactic, the new free variable name is provided here
+  -- In case of the `have` and `let` tactics, the new free variable name is
+  -- provided here
   binderName?: Option String := .none
 
   deriving Lean.FromJson
 structure GoalTacticResult where
   -- The next goal state id. Existence of this field shows success
   nextStateId?: Option Nat          := .none
-  -- If the array is empty, it shows the goals have been fully resolved.
-  goals?: Option (Array Goal)          := .none
+  -- If the array is empty, it shows the goals have been fully resolved. If this
+  -- is .none, there has been a tactic error.
+  goals?: Option (Array Goal)       := .none
 
-  -- Existence of this field shows tactic execution failure
-  tacticErrors?: Option (Array String) := .none
+  messages? : Option (Array Lean.SerialMessage) := .some #[]
 
   -- Existence of this field shows the tactic parsing has failed
-  parseError?: Option String := .none
+  parseError? : Option String       := .none
+
+  hasSorry : Bool := false
+  hasUnsafe : Bool := false
   deriving Lean.ToJson
 structure GoalContinue where
   -- State from which the continuation acquires the context
@@ -301,8 +313,8 @@ structure GoalPrint where
 
   -- Print root?
   rootExpr?: Option Bool := .some False
-  -- Print the parent expr?
-  parentExpr?: Option Bool := .some False
+  -- Print the parent expressions
+  parentExprs?: Option Bool := .some False
   -- Print goals?
   goals?: Option Bool := .some False
   -- Print values of extra mvars?
@@ -312,9 +324,13 @@ structure GoalPrintResult where
   -- The root expression
   root?: Option Expression := .none
   -- The filling expression of the parent goal
-  parent?: Option Expression := .none
+  parentExprs?: Option (List (Option Expression)) := .none
   goals: Array Goal := #[]
   extraMVars: Array Expression := #[]
+
+  rootHasSorry : Bool := false
+  rootHasUnsafe : Bool := false
+  rootHasMVar : Bool := true
   deriving Lean.ToJson
 
 -- Diagnostic Options, not available in REPL
@@ -346,8 +362,12 @@ structure FrontendProcess where
   -- One of these two must be supplied: Either supply the file name or the content.
   fileName?: Option String := .none
   file?: Option String := .none
-  -- collect tactic invocations
-  invocations: Bool := false
+  -- Whether to read the header
+  readHeader : Bool := false
+  -- Alter the REPL environment after the compilation units.
+  inheritEnv : Bool := false
+  -- collect tactic invocations and output to a given file
+  invocations?: Option String := .none
   -- collect `sorry`s
   sorrys: Bool := false
   -- collect type errors
@@ -367,9 +387,9 @@ structure InvokedTactic where
 structure CompilationUnit where
   -- String boundaries of compilation units
   boundary: (Nat × Nat)
-  messages: Array String := #[]
-  -- Tactic invocations
-  invocations?: Option (List InvokedTactic) := .none
+  messages: Array Lean.SerialMessage := #[]
+  -- Number of tactic invocations
+  nInvocations?: Option Nat := .none
   goalStateId?: Option Nat := .none
   goals?: Option (Array Goal) := .none
   -- Code segments which generated the goals
@@ -381,7 +401,16 @@ structure CompilationUnit where
 structure FrontendProcessResult where
   units: List CompilationUnit
   deriving Lean.ToJson
+structure FrontendDataUnit where
+  invocations? : Option (List Protocol.InvokedTactic) := .none
+  deriving Lean.ToJson
+structure FrontendData where
+  units : List FrontendDataUnit
+  deriving Lean.ToJson
 
-abbrev CR α := Except InteractionError α
+abbrev FallibleT := ExceptT InteractionError
+
+abbrev throw {m : Type v → Type w} [MonadExceptOf InteractionError m] {α : Type v} (e : InteractionError) : m α :=
+  throwThe InteractionError e
 
 end Pantograph.Protocol

Pantograph/Serial.lean

@@ -1,19 +1,19 @@
-import Lean.Environment
-import Lean.Replay
-import Init.System.IOError
-import Std.Data.HashMap
 import Pantograph.Goal
 
+import Lean.Environment
+import Lean.Replay
+import Std.Data.HashMap
+
+open Lean
+
 /-!
-Input/Output functions
+Input/Output functions borrowed from REPL
 
 # Pickling and unpickling objects
 
 By abusing `saveModuleData` and `readModuleData` we can pickle and unpickle objects to disk.
 -/
 
-open Lean
-
 namespace Pantograph
 
 /--
@@ -46,6 +46,21 @@ unsafe def withUnpickle [Monad m] [MonadLiftT IO m] {α β : Type}
   region.free
   pure r
 
+abbrev ConstArray := Array (Name × ConstantInfo)
+abbrev DistilledEnvironment := Array Import × ConstArray
+
+/-- Boil an environment down to minimal components -/
+def distillEnvironment (env : Environment) (background? : Option Environment := .none)
+  : DistilledEnvironment :=
+  let filter : Name → Bool := match background? with
+    | .some env => (¬ env.contains ·)
+    | .none => λ _ => true
+  let constants : ConstArray := env.constants.map₂.foldl (init := #[]) λ acc name info =>
+    if filter name then
+      acc.push (name, info)
+    else
+      acc
+  (env.header.imports, constants)
 /--
 Pickle an `Environment` to disk.
 
@@ -56,15 +71,23 @@ and when unpickling, we build a fresh `Environment` from the imports,
 and then add the new constants.
 -/
 @[export pantograph_env_pickle_m]
-def environmentPickle (env : Environment) (path : System.FilePath) : IO Unit :=
-  Pantograph.pickle path (env.header.imports, env.constants.map₂)
+def environmentPickle (env : Environment) (path : System.FilePath) (background? : Option Environment := .none)
+  : IO Unit :=
+  pickle path $ distillEnvironment env background?
+
+deriving instance BEq for Import
 
 def resurrectEnvironment
-  (imports : Array Import)
-  (map₂ : PHashMap Name ConstantInfo)
+  (distilled : DistilledEnvironment)
+  (background? : Option Environment := .none)
   : IO Environment := do
-  let env ← importModules imports {} 0
-  env.replay (Std.HashMap.ofList map₂.toList)
+  let (imports, constArray) := distilled
+  let env ← match background? with
+    | .none => importModules imports {} 0 (loadExts := true)
+    | .some env =>
+      assert! env.imports == imports
+      pure env
+  env.replay (Std.HashMap.ofList constArray.toList)
 /--
 Unpickle an `Environment` from disk.
 
@@ -72,90 +95,103 @@ We construct a fresh `Environment` with the relevant imports,
 and then replace the new constants.
 -/
 @[export pantograph_env_unpickle_m]
-def environmentUnpickle (path : System.FilePath) : IO (Environment × CompactedRegion) := unsafe do
-  let ((imports, map₂), region) ← Pantograph.unpickle (Array Import × PHashMap Name ConstantInfo) path
-  return (← resurrectEnvironment imports map₂, region)
+def environmentUnpickle (path : System.FilePath) (background? : Option Environment := .none)
+  : IO (Environment × CompactedRegion) := unsafe do
+  let (distilled, region) ← unpickle (Array Import × ConstArray) path
+  return (← resurrectEnvironment distilled background?, region)
 
 
-open Lean.Core in
+/-- `CoreM`'s state, with information irrelevant to pickling masked out -/
 structure CompactCoreState where
   -- env             : Environment
   nextMacroScope  : MacroScope     := firstFrontendMacroScope + 1
   ngen            : NameGenerator  := {}
+  auxDeclNGen     : DeclNameGenerator := {}
   -- traceState      : TraceState     := {}
   -- cache           : Cache     := {}
   -- messages        : MessageLog     := {}
   -- infoState       : Elab.InfoState := {}
 
+structure CompactGoalState where
+  env : DistilledEnvironment
+
+  core : CompactCoreState
+  meta : Meta.State
+  «elab»: Elab.Term.State
+  tactic: Elab.Tactic.State
+  root: MVarId
+  parentMVars: List MVarId
+  fragments: FragmentMap
+
+/-- Pickles a goal state by taking its diff relative to a background
+environment. This function eliminates all `MessageData` from synthetic
+metavariables, because these `MessageData` objects frequently carry closures,
+which cannot be pickled. If there is no synthetic metavariable, this would not
+cause a difference. -/
 @[export pantograph_goal_state_pickle_m]
-def goalStatePickle (goalState : GoalState) (path : System.FilePath) : IO Unit :=
+def goalStatePickle (goalState : GoalState) (path : System.FilePath) (background? : Option Environment := .none)
+  : IO Unit :=
   let {
     savedState := {
       term := {
         meta := {
           core := {
-            env, nextMacroScope, ngen, ..
+            env, nextMacroScope, ngen, auxDeclNGen, ..
           },
           meta,
         }
-        «elab»,
+        «elab» := «elab»@{
+          syntheticMVars, ..
+        },
       },
       tactic
     }
     root,
-    parentMVar?,
-    convMVar?,
-    calcPrevRhs?,
+    parentMVars,
+    fragments,
   } := goalState
-  Pantograph.pickle path (
-    env.constants.map₂,
+  -- Delete `MessageData`s
+  let syntheticMVars : MVarIdMap _ := syntheticMVars.fold (init := .empty) λ acc key val =>
+    let kind := match val.kind with
+      | .typeClass _ => .typeClass .none
+      | .coe header? expectedType e f? _ => .coe header? expectedType e f? .none
+      | k => k
+    acc.insert key { val with kind }
+  pickle path ({
+    env := distillEnvironment env background?,
 
-    ({ nextMacroScope, ngen } : CompactCoreState),
+    core := ({ nextMacroScope, ngen, auxDeclNGen } : CompactCoreState),
     meta,
-    «elab»,
+    «elab» := { «elab» with syntheticMVars },
     tactic,
 
     root,
-    parentMVar?,
-    convMVar?,
-    calcPrevRhs?,
-  )
+    parentMVars,
+    fragments,
+  } : CompactGoalState)
 
 @[export pantograph_goal_state_unpickle_m]
-def goalStateUnpickle (path : System.FilePath) (env : Environment)
+def goalStateUnpickle (path : System.FilePath) (background? : Option Environment := .none)
     : IO (GoalState × CompactedRegion) := unsafe do
-  let ((
-    map₂,
+  let ({
+    env,
 
-    compactCore,
+    core,
     meta,
     «elab»,
     tactic,
 
     root,
-    parentMVar?,
-    convMVar?,
-    calcPrevRhs?,
-  ), region) ← Pantograph.unpickle (
-    PHashMap Name ConstantInfo ×
-
-    CompactCoreState ×
-    Meta.State ×
-    Elab.Term.State ×
-    Elab.Tactic.State ×
-
-    MVarId ×
-    Option MVarId ×
-    Option (MVarId × MVarId × List MVarId) ×
-    Option (MVarId × Expr)
-  ) path
-  let env ← env.replay (Std.HashMap.ofList map₂.toList)
+    parentMVars,
+    fragments,
+  }, region) ← Pantograph.unpickle CompactGoalState path
+  let env ← resurrectEnvironment env background?
   let goalState := {
     savedState := {
       term := {
         meta := {
           core := {
-            compactCore with
+            core with
             passedHeartbeats := 0,
             env,
           },
@@ -166,9 +202,8 @@ def goalStateUnpickle (path : System.FilePath) (env : Environment)
       tactic,
     },
     root,
-    parentMVar?,
-    convMVar?,
-    calcPrevRhs?,
+    parentMVars,
+    fragments,
   }
   return (goalState, region)
 

Pantograph/Tactic.lean

@@ -1,5 +1,3 @@
 import Pantograph.Tactic.Assign
-import Pantograph.Tactic.Congruence
-import Pantograph.Tactic.MotivatedApply
-import Pantograph.Tactic.NoConfuse
+import Pantograph.Tactic.Fragment
 import Pantograph.Tactic.Prograde

Pantograph/Tactic/Assign.lean

@@ -28,15 +28,16 @@ def evalAssign : Elab.Tactic.Tactic := fun stx => Elab.Tactic.withMainContext do
   Elab.Tactic.replaceMainGoal nextGoals
 
 def sorryToHole (src : Expr) : StateRefT (List MVarId) MetaM Expr := do
-  Meta.transform src λ
-    | .app (.app (.const ``sorryAx ..) type) .. => do
-      let type ← instantiateMVars type
+  Meta.transform src λ expr =>
+    if expr.isSorry then do
+      let type ← instantiateMVars (expr.getArg! 0 |>.bindingBody!)
       if type.hasSorry then
         throwError s!"Coupling is not allowed in draft tactic: {← Meta.ppExpr type}"
       let mvar ← Meta.mkFreshExprSyntheticOpaqueMVar type
       modify (mvar.mvarId! :: .)
       pure $ .done mvar
-    | _ => pure .continue
+    else
+      pure .continue
 
 -- Given a complete (no holes) expression, extract the sorry's from it and convert them into goals.
 def draft (goal : MVarId) (expr : Expr) : MetaM (List MVarId) := do

Pantograph/Tactic/Congruence.lean

@@ -1,98 +0,0 @@
-import Lean
-
-open Lean
-
-namespace Pantograph.Tactic
-
-def congruenceArg (mvarId: MVarId): MetaM (List MVarId) := mvarId.withContext do
-  mvarId.checkNotAssigned `Pantograph.Tactic.congruenceArg
-  let target ← mvarId.getType
-  let .some (β, _, _) := (← instantiateMVars target).eq? | throwError "Goal is not an Eq"
-  let userName := (← mvarId.getDecl).userName
-
-  let u ← Meta.mkFreshLevelMVar
-  let α ← Meta.mkFreshExprSyntheticOpaqueMVar (mkSort u)
-    (tag := userName ++ `α)
-  let f ← Meta.mkFreshExprSyntheticOpaqueMVar (.forallE .anonymous α β .default)
-    (tag := userName ++ `f)
-  let a₁ ← Meta.mkFreshExprSyntheticOpaqueMVar α
-    (tag := userName ++ `a₁)
-  let a₂ ← Meta.mkFreshExprSyntheticOpaqueMVar α
-    (tag := userName ++ `a₂)
-  let h ← Meta.mkFreshExprSyntheticOpaqueMVar (← Meta.mkEq a₁ a₂)
-    (tag := userName ++ `h)
-  let conduitType ← Meta.mkEq (← Meta.mkEq (.app f a₁) (.app f a₂)) target
-  let conduit ← Meta.mkFreshExprSyntheticOpaqueMVar conduitType
-    (tag := userName ++ `conduit)
-  mvarId.assign $ ← Meta.mkEqMP conduit (← Meta.mkCongrArg f h)
-  let result := [α, a₁, a₂, f,  h, conduit]
-  return result.map (·.mvarId!)
-
-def evalCongruenceArg: Elab.Tactic.TacticM Unit := do
-  let goal ← Elab.Tactic.getMainGoal
-  let nextGoals ← congruenceArg goal
-  Elab.Tactic.replaceMainGoal nextGoals
-
-def congruenceFun (mvarId: MVarId): MetaM (List MVarId) := mvarId.withContext do
-  mvarId.checkNotAssigned `Pantograph.Tactic.congruenceFun
-  let target ← mvarId.getType
-  let .some (β, _, _) := (← instantiateMVars target).eq? | throwError "Goal is not an Eq"
-  let userName := (← mvarId.getDecl).userName
-  let u ← Meta.mkFreshLevelMVar
-  let α ← Meta.mkFreshExprSyntheticOpaqueMVar (mkSort u)
-    (tag := userName ++ `α)
-  let fType := .forallE .anonymous α β .default
-  let f₁ ← Meta.mkFreshExprSyntheticOpaqueMVar fType
-    (tag := userName ++ `f₁)
-  let f₂ ← Meta.mkFreshExprSyntheticOpaqueMVar fType
-    (tag := userName ++ `f₂)
-  let a ← Meta.mkFreshExprSyntheticOpaqueMVar α
-    (tag := userName ++ `a)
-  let h ← Meta.mkFreshExprSyntheticOpaqueMVar (← Meta.mkEq f₁ f₂)
-    (tag := userName ++ `h)
-  let conduitType ← Meta.mkEq (← Meta.mkEq (.app f₁ a) (.app f₂ a)) target
-  let conduit ← Meta.mkFreshExprSyntheticOpaqueMVar conduitType
-    (tag := userName ++ `conduit)
-  mvarId.assign $ ← Meta.mkEqMP conduit (← Meta.mkCongrFun h a)
-  let result := [α, f₁, f₂, h, a, conduit]
-  return result.map (·.mvarId!)
-
-def evalCongruenceFun: Elab.Tactic.TacticM Unit := do
-  let goal ← Elab.Tactic.getMainGoal
-  let nextGoals ← congruenceFun goal
-  Elab.Tactic.replaceMainGoal nextGoals
-
-def congruence (mvarId: MVarId): MetaM (List MVarId) := mvarId.withContext do
-  mvarId.checkNotAssigned `Pantograph.Tactic.congruence
-  let target ← mvarId.getType
-  let .some (β, _, _) := (← instantiateMVars target).eq? | throwError "Goal is not an Eq"
-  let userName := (← mvarId.getDecl).userName
-  let u ← Meta.mkFreshLevelMVar
-  let α ← Meta.mkFreshExprSyntheticOpaqueMVar (mkSort u)
-    (tag := userName ++ `α)
-  let fType := .forallE .anonymous α β .default
-  let f₁ ← Meta.mkFreshExprSyntheticOpaqueMVar fType
-    (tag := userName ++ `f₁)
-  let f₂ ← Meta.mkFreshExprSyntheticOpaqueMVar fType
-    (tag := userName ++ `f₂)
-  let a₁ ← Meta.mkFreshExprSyntheticOpaqueMVar α
-    (tag := userName ++ `a₁)
-  let a₂ ← Meta.mkFreshExprSyntheticOpaqueMVar α
-    (tag := userName ++ `a₂)
-  let h₁ ← Meta.mkFreshExprSyntheticOpaqueMVar (← Meta.mkEq f₁ f₂)
-    (tag := userName ++ `h₁)
-  let h₂ ← Meta.mkFreshExprSyntheticOpaqueMVar (← Meta.mkEq a₁ a₂)
-    (tag := userName ++ `h₂)
-  let conduitType ← Meta.mkEq (← Meta.mkEq (.app f₁ a₁) (.app f₂ a₂)) target
-  let conduit ← Meta.mkFreshExprSyntheticOpaqueMVar conduitType
-    (tag := userName ++ `conduit)
-  mvarId.assign $ ← Meta.mkEqMP conduit (← Meta.mkCongr h₁ h₂)
-  let result := [α, f₁, f₂, a₁, a₂, h₁, h₂, conduit]
-  return result.map (·.mvarId!)
-
-def evalCongruence: Elab.Tactic.TacticM Unit := do
-  let goal ← Elab.Tactic.getMainGoal
-  let nextGoals ← congruence goal
-  Elab.Tactic.replaceMainGoal nextGoals
-
-end Pantograph.Tactic

Pantograph/Tactic/Fragment.lean

@@ -0,0 +1,189 @@
+/- Fragmented tactics are the tactics which can give incremental feedback and
+whose integrity as a block is crucial to its operation. e.g. `calc` or `conv`.
+Here, a unified system handles all fragments.
+
+Inside a tactic fragment, the parser category may be different. An incomplete
+fragmented tactic may not be elaboratable..
+
+In line with continuation/resumption paradigms, the exit function of a fragment
+tactic is responsible for resuming incomplete goals with fragments. For example,
+when a conversion tactic finishes, the sentinels should resume the root of the
+conversion tactic goal. The user cannot be expected to execute this resumption,
+since the root is automatically dormanted at the entry of the conversion tactic
+mode.
+-/
+import Lean.Meta
+import Lean.Elab
+
+open Lean
+
+namespace Pantograph
+
+inductive Fragment where
+  | calc (prevRhs? : Option Expr)
+  | conv (rhs : MVarId)
+  -- This goal is spawned from a `conv`
+  | convSentinel (parent : MVarId)
+  deriving BEq, Inhabited
+
+abbrev FragmentMap := Std.HashMap MVarId Fragment
+def FragmentMap.empty : FragmentMap := Std.HashMap.emptyWithCapacity 2
+protected def FragmentMap.filter (map : FragmentMap) (pred : MVarId → Fragment → Bool) : FragmentMap :=
+  map.fold (init := FragmentMap.empty) λ acc mvarId fragment =>
+    if pred mvarId fragment then
+      acc.insert mvarId fragment
+    else
+      acc
+
+protected def Fragment.map (fragment : Fragment) (mapExpr : Expr → CoreM Expr) : CoreM Fragment :=
+  let mapMVar (mvarId : MVarId) : CoreM MVarId :=
+    return (← mapExpr (.mvar mvarId)) |>.mvarId!
+  match fragment with
+  | .calc prevRhs? => return .calc (← prevRhs?.mapM mapExpr)
+  | .conv rhs => do
+    let rhs' ← mapMVar rhs
+    return .conv rhs'
+  | .convSentinel parent => do
+    return .convSentinel (← mapMVar parent)
+
+protected def Fragment.enterCalc : Fragment := .calc .none
+protected def Fragment.enterConv : Elab.Tactic.TacticM FragmentMap := do
+  let goal ← Elab.Tactic.getMainGoal
+  goal.checkNotAssigned `GoalState.conv
+  let (rhs, newGoal) ← goal.withContext do
+    let target ← instantiateMVars (← goal.getType)
+    let (rhs, newGoal) ← Elab.Tactic.Conv.mkConvGoalFor target
+    pure (rhs.mvarId!, newGoal.mvarId!)
+  Elab.Tactic.replaceMainGoal [newGoal]
+  return FragmentMap.empty
+    |>.insert goal (.conv rhs)
+    |>.insert newGoal (.convSentinel goal)
+
+protected partial def Fragment.exit (fragment : Fragment) (goal : MVarId) (fragments : FragmentMap)
+  : Elab.Tactic.TacticM FragmentMap :=
+  match fragment with
+  | .calc .. => do
+    Elab.Tactic.setGoals [goal]
+    return fragments.erase goal
+  | .conv rhs => do
+    let goals := (← Elab.Tactic.getGoals).filter λ descendant =>
+      match fragments[descendant]? with
+      | .some s => (.convSentinel goal) == s
+      | _ => false -- Not a conv goal from this
+    -- Close all existing goals with `refl`
+    for mvarId in goals do
+      liftM <| mvarId.refl <|> mvarId.inferInstance <|> pure ()
+    unless (← goals.filterM (·.isAssignedOrDelayedAssigned)).isEmpty do
+      throwError "convert tactic failed, there are unsolved goals\n{Elab.goalsToMessageData (goals)}"
+
+    -- Ensure the meta tactic runs on `goal` even if its dormant by forcing resumption
+    Elab.Tactic.setGoals $ goal :: (← Elab.Tactic.getGoals)
+    let targetNew ← instantiateMVars (.mvar rhs)
+    let proof ← instantiateMVars (.mvar goal)
+
+    Elab.Tactic.liftMetaTactic1 (·.replaceTargetEq targetNew proof)
+
+    -- Try to solve maiinline by rfl
+    let mvarId ← Elab.Tactic.getMainGoal
+    liftM <| mvarId.refl <|> mvarId.inferInstance <|> pure ()
+    Elab.Tactic.pruneSolvedGoals
+    return fragments.filter λ mvarId fragment =>
+      !(mvarId == goal || fragment == .convSentinel goal)
+  | .convSentinel parent =>
+    let parentFragment := fragments[parent]!
+    parentFragment.exit parent (fragments.erase goal)
+
+protected def Fragment.step (fragment : Fragment) (goal : MVarId) (s : String) (map : FragmentMap)
+  : Elab.Tactic.TacticM FragmentMap := goal.withContext do
+  assert! ¬ (← goal.isAssigned)
+  match fragment with
+  | .calc prevRhs? => do
+    let .ok stx := Parser.runParserCategory
+      (env := ← getEnv)
+      (catName := `term)
+      (input := s)
+      (fileName := ← getFileName) | throwError s!"Failed to parse calc element {s}"
+    let `(term|$pred) := stx
+    let decl ← goal.getDecl
+    let target ← instantiateMVars decl.type
+    let tag := decl.userName
+
+    let mut step ← Elab.Term.elabType <| ← do
+      if let some prevRhs := prevRhs? then
+        Elab.Term.annotateFirstHoleWithType pred (← Meta.inferType prevRhs)
+      else
+        pure pred
+
+    let some (_, lhs, rhs) ← Elab.Term.getCalcRelation? step |
+      throwErrorAt pred "invalid 'calc' step, relation expected{indentExpr step}"
+    if let some prevRhs := prevRhs? then
+      unless ← Meta.isDefEqGuarded lhs prevRhs do
+        throwErrorAt pred "invalid 'calc' step, left-hand-side is{indentD m!"{lhs} : {← Meta.inferType lhs}"}\nprevious right-hand-side is{indentD m!"{prevRhs} : {← Meta.inferType prevRhs}"}"
+
+    -- Creates a mvar to represent the proof that the calc tactic solves the
+    -- current branch
+    -- In the Lean `calc` tactic this is gobbled up by
+    -- `withCollectingNewGoalsFrom`
+    let mut proof ← Meta.mkFreshExprMVarAt (← getLCtx) (← Meta.getLocalInstances) step
+      (userName := tag ++ `calc)
+    let mvarBranch := proof.mvarId!
+
+    let mut proofType ← Meta.inferType proof
+    let mut remainder? := Option.none
+
+    -- The calc tactic either solves the main goal or leaves another relation.
+    -- Replace the main goal, and save the new goal if necessary
+    unless ← Meta.isDefEq proofType target do
+      let rec throwFailed :=
+        throwError "'calc' tactic failed, has type{indentExpr proofType}\nbut it is expected to have type{indentExpr target}"
+      let some (_, _, rhs) ← Elab.Term.getCalcRelation? proofType | throwFailed
+      let some (r, _, rhs') ← Elab.Term.getCalcRelation? target | throwFailed
+      let lastStep := mkApp2 r rhs rhs'
+      let lastStepGoal ← Meta.mkFreshExprSyntheticOpaqueMVar lastStep tag
+      (proof, proofType) ← Elab.Term.mkCalcTrans proof proofType lastStepGoal lastStep
+      unless ← Meta.isDefEq proofType target do throwFailed
+      remainder? := .some lastStepGoal.mvarId!
+    goal.assign proof
+    let map := map.erase goal
+
+    let goals := [ mvarBranch ] ++ remainder?.toList
+    Elab.Tactic.setGoals goals
+    match remainder? with
+    | .some goal => return map.insert goal $ .calc (.some rhs)
+    | .none => return map
+  | .conv .. => do
+    throwError "Direct operation on conversion tactic parent goal is not allowed"
+  | fragment@(.convSentinel parent) => do
+    assert! isLHSGoal? (← goal.getType) |>.isSome
+    let tactic ← match Parser.runParserCategory
+      (env := ← MonadEnv.getEnv)
+      (catName := `conv)
+      (input := s)
+      (fileName := ← getFileName) with
+      | .ok stx => pure $ stx
+      | .error error => throwError error
+    let oldGoals ← Elab.Tactic.getGoals
+    -- Label newly generated goals as conv sentinels
+    Elab.Tactic.evalTactic tactic
+    let newConvGoals ← (← Elab.Tactic.getUnsolvedGoals).filterM λ g => do
+      -- conv tactic might generate non-conv goals
+      if oldGoals.contains g then
+        return false
+      return isLHSGoal? (← g.getType) |>.isSome
+    -- Conclude the conv by exiting the parent fragment if new goals is empty
+    if newConvGoals.isEmpty then
+      let hasSiblingFragment := map.fold (init := false) λ flag _ fragment =>
+        if flag then
+          true
+        else match fragment with
+          | .convSentinel parent' => parent == parent'
+          | _ => false
+      if ¬ hasSiblingFragment then
+        -- This fragment must exist since we have conv goals
+        let parentFragment := map[parent]!
+        -- All descendants exhausted. Exit from the parent conv.
+        return ← parentFragment.exit parent map
+    return newConvGoals.foldl (init := map) λ acc g =>
+      acc.insert g fragment
+
+end Pantograph

Pantograph/Tactic/MotivatedApply.lean

@@ -1,106 +0,0 @@
-import Lean
-
-open Lean
-
-namespace Pantograph.Tactic
-
-def getForallArgsBody: Expr → List Expr × Expr
-  | .forallE _ d b _ =>
-    let (innerArgs, innerBody) := getForallArgsBody b
-    (d :: innerArgs, innerBody)
-  | e => ([], e)
-
-def replaceForallBody: Expr → Expr → Expr
-  | .forallE param domain body binderInfo, target =>
-    let body := replaceForallBody body target
-    .forallE param domain body binderInfo
-  | _, target => target
-
-structure RecursorWithMotive where
-  args: List Expr
-  body: Expr
-
-  -- .bvar index for the motive and major from the body
-  iMotive: Nat
-
-namespace RecursorWithMotive
-
-protected def nArgs (info: RecursorWithMotive): Nat := info.args.length
-
-protected def getMotiveType (info: RecursorWithMotive): Expr :=
-  let level := info.nArgs - info.iMotive - 1
-  let a := info.args.get! level
-  a
-
-protected def surrogateMotiveType (info: RecursorWithMotive) (mvars: Array Expr) (resultant: Expr): MetaM Expr := do
-  let motiveType := Expr.instantiateRev info.getMotiveType mvars
-  let resultantType ← Meta.inferType resultant
-  return replaceForallBody motiveType resultantType
-
-protected def conduitType (info: RecursorWithMotive) (mvars: Array Expr) (resultant: Expr): MetaM Expr := do
-  let motiveCall := Expr.instantiateRev info.body mvars
-  Meta.mkEq motiveCall resultant
-
-end RecursorWithMotive
-
-def getRecursorInformation (recursorType: Expr): Option RecursorWithMotive := do
-  let (args, body) := getForallArgsBody recursorType
-  if ¬ body.isApp then
-    .none
-  let iMotive ← match body.getAppFn with
-    | .bvar iMotive => pure iMotive
-    | _ => .none
-  return {
-    args,
-    body,
-    iMotive,
-  }
-
-def collectMotiveArguments (forallBody: Expr): SSet Nat :=
-  match forallBody with
-  | .app (.bvar i) _ => SSet.empty.insert i
-  | _ => SSet.empty
-
-/-- Applies a symbol of the type `∀ (motive: α → Sort u) (a: α)..., (motive α)` -/
-def motivatedApply (mvarId: MVarId) (recursor: Expr) : MetaM (Array Meta.InductionSubgoal) := mvarId.withContext do
-  mvarId.checkNotAssigned `Pantograph.Tactic.motivatedApply
-  let recursorType ← Meta.inferType recursor
-  let resultant ← mvarId.getType
-  let tag ← mvarId.getTag
-
-  let info ← match getRecursorInformation recursorType with
-    | .some info => pure info
-    | .none => throwError "Recursor return type does not correspond with the invocation of a motive: {← Meta.ppExpr recursorType}"
-
-  let rec go (i: Nat) (prev: Array Expr): MetaM (Array Expr) := do
-    if i ≥ info.nArgs then
-      return prev
-    else
-      let argType := info.args.get! i
-      -- If `argType` has motive references, its goal needs to be placed in it
-      let argType := argType.instantiateRev prev
-      let bvarIndex := info.nArgs - i - 1
-      let argGoal ← if bvarIndex = info.iMotive then
-          let surrogateMotiveType ← info.surrogateMotiveType prev resultant
-          Meta.mkFreshExprSyntheticOpaqueMVar surrogateMotiveType (tag := tag ++ `motive)
-        else
-          Meta.mkFreshExprSyntheticOpaqueMVar argType (tag := .anonymous)
-      let prev :=  prev ++ [argGoal]
-      go (i + 1) prev
-    termination_by info.nArgs - i
-  let mut newMVars ← go 0 #[]
-
-  -- Create the conduit type which proves the result of the motive is equal to the goal
-  let conduitType ← info.conduitType newMVars resultant
-  let goalConduit ← Meta.mkFreshExprSyntheticOpaqueMVar conduitType (tag := `conduit)
-  mvarId.assign $ ← Meta.mkEqMP goalConduit (mkAppN recursor newMVars)
-  newMVars := newMVars ++ [goalConduit]
-
-  return newMVars.map (λ mvar => { mvarId := mvar.mvarId!})
-
-def evalMotivatedApply : Elab.Tactic.Tactic := fun stx => Elab.Tactic.withMainContext do
-  let recursor ← Elab.Term.elabTerm (stx := stx) .none
-  let nextGoals ← motivatedApply (← Elab.Tactic.getMainGoal) recursor
-  Elab.Tactic.replaceMainGoal $ nextGoals.toList.map (·.mvarId)
-
-end Pantograph.Tactic

Pantograph/Tactic/NoConfuse.lean

@@ -1,22 +0,0 @@
-import Lean
-
-open Lean
-
-namespace Pantograph.Tactic
-
-def noConfuse (mvarId: MVarId) (h: Expr): MetaM Unit := mvarId.withContext do
-  mvarId.checkNotAssigned `Pantograph.Tactic.noConfuse
-  let target ← mvarId.getType
-  let noConfusion ← Meta.mkNoConfusion (target := target) (h := h)
-
-  unless ← Meta.isDefEq (← Meta.inferType noConfusion) target do
-    throwError "invalid noConfuse call: The resultant type {← Meta.ppExpr $ ← Meta.inferType noConfusion} cannot be unified with {← Meta.ppExpr target}"
-  mvarId.assign noConfusion
-
-def evalNoConfuse: Elab.Tactic.Tactic := λ stx => do
-  let goal ← Elab.Tactic.getMainGoal
-  let h ← goal.withContext $ Elab.Term.elabTerm (stx := stx) .none
-  noConfuse goal h
-  Elab.Tactic.replaceMainGoal []
-
-end Pantograph.Tactic

Pantograph/Version.lean

@@ -1,6 +1,6 @@
 namespace Pantograph
 
 @[export pantograph_version]
-def version := "0.2.25"
+def version := "0.3.5"
 
 end Pantograph

README.md

@@ -17,101 +17,33 @@ nix build .#{sharedLib,executable}
 ```
 to build either the shared library or executable.
 
-Install `lake` and `lean` fixed to the version of the `lean-toolchain` file, and
-run
+For non-Nix users, install `lake` and `lean` fixed to the version of the
+`lean-toolchain` file, and run
 
 ``` sh
 lake build
 ```
-This builds the executable in `.lake/build/bin/pantograph-repl`.
+This builds the executable in `.lake/build/bin/repl`.
 
-## Executable Usage
+### Executable Usage
 
-``` sh
-pantograph-repl MODULES|LEAN_OPTIONS
-```
+The default build target is a Read-Eval-Print-Loop (REPL). See [REPL
+Documentation](./doc/repl.md)
 
-The `pantograph-repl` executable must be run with a list of modules to import.
-It can also accept lean options of the form `--key=value` e.g. `--pp.raw=true`.
+Another executable is the `tomograph`, which processes a Lean file and displays
+syntax or elaboration level data.
 
-The REPL loop accepts commands as single-line JSON inputs and outputs either an
-`Error:` (indicating malformed command) or a JSON return value indicating the
-result of a command execution.  The command can be passed in one of two formats
-```
-command { ... }
-{ "cmd": command, "payload": ... }
-```
-The list of available commands can be found in `Pantograph/Protocol.lean` and below. An
-empty command aborts the REPL.
-
-
-Example: (~5k symbols)
-```
-$ pantograph Init
-env.catalog
-env.inspect {"name": "Nat.le_add_left"}
-```
-Example with `mathlib4` (~90k symbols, may stack overflow, see troubleshooting)
-```
-$ pantograph Mathlib.Analysis.Seminorm
-env.catalog
-```
-Example proving a theorem: (alternatively use `goal.start {"copyFrom": "Nat.add_comm"}`) to prime the proof
-```
-$ pantograph Init
-goal.start {"expr": "∀ (n m : Nat), n + m = m + n"}
-goal.tactic {"stateId": 0, "goalId": 0, "tactic": "intro n m"}
-goal.tactic {"stateId": 1, "goalId": 0, "tactic": "assumption"}
-goal.delete {"stateIds": [0]}
-stat {}
-goal.tactic {"stateId": 1, "goalId": 0, "tactic": "rw [Nat.add_comm]"}
-stat
-```
-where the application of `assumption` should lead to a failure.
-
-For a list of commands, see [REPL Documentation](doc/repl.md).
-
-### Project Environment
-
-To use Pantograph in a project environment, setup the `LEAN_PATH` environment
-variable so it contains the library path of lean libraries. The libraries must
-be built in advance. For example, if `mathlib4` is stored at `../lib/mathlib4`,
-the environment might be setup like this:
-
-``` sh
-LIB="../lib"
-LIB_MATHLIB="$LIB/mathlib4/.lake"
-export LEAN_PATH="$LIB/mathlib4/build/lib:$LIB_MATHLIB/aesop/build/lib:$LIB_MATHLIB/Qq/build/lib:$LIB_MATHLIB/std/build/lib"
-
-LEAN_PATH=$LEAN_PATH build/bin/pantograph $@
-```
-The `$LEAN_PATH` executable of any project can be extracted by
-``` sh
-lake env printenv LEAN_PATH
-```
-
-### Troubleshooting
-
-If lean encounters stack overflow problems when printing catalog, execute this before running lean:
-```sh
-ulimit -s unlimited
-```
-
-## Library Usage
+### Library Usage
 
 `Pantograph/Library.lean` exposes a series of interfaces which allow FFI call
-with `Pantograph` which mirrors the REPL commands above. It is recommended to
-call Pantograph via this FFI since it provides a tremendous speed up.
-
-The executable can be used as-is, but linking against the shared library
-requires the presence of `lean-all`. Note that there isn't a 1-1 correspondence
-between executable (REPL) commands and library functions.
+with `Pantograph` which mirrors the REPL commands above. Note that there isn't a
+1-1 correspondence between executable (REPL) commands and library functions.
 
 Inject any project path via the `pantograph_init_search` function.
 
-## Developing
+## Development
 
-A Lean development shell is provided in the Nix flake.
+A Lean development shell is provided in the Nix flake. Nix usage is optional.
 
 ### Testing
 

Repl.lean

@@ -6,19 +6,34 @@ namespace Pantograph.Repl
 open Lean
 
 structure Context where
+  coreContext : Core.Context
+  -- If true, the environment will change after running `CoreM`
+  inheritEnv : Bool := false
 
 /-- Stores state of the REPL -/
 structure State where
-  options: Protocol.Options := {}
-  nextId: Nat := 0
-  goalStates: Std.HashMap Nat GoalState := Std.HashMap.empty
+  options : Protocol.Options := {}
+  nextId : Nat := 0
+  goalStates : Std.HashMap Nat GoalState := Std.HashMap.emptyWithCapacity
 
-/-- Main state monad for executing commands -/
-abbrev MainM := ReaderT Context $ StateRefT State CoreM
-/-- Fallible subroutine return type -/
-abbrev CR α := Except Protocol.InteractionError α
+  env : Environment
+  -- Parser state
+  scope : Elab.Command.Scope := { header := "" }
 
-def newGoalState (goalState: GoalState) : MainM Nat := do
+/-- Main monad for executing commands -/
+abbrev MainM := ReaderT Context $ StateRefT State IO
+/-- Main with possible exception -/
+abbrev EMainM := Protocol.FallibleT $ ReaderT Context $ StateRefT State IO
+def getMainState : MainM State := get
+
+instance : MonadEnv MainM where
+  getEnv := return (← get).env
+  modifyEnv f := modify fun s => { s with env := f s.env  }
+
+def withInheritEnv [Monad m] [MonadWithReaderOf Context m] [MonadLift MainM m] { α } (z : m α) : m α := do
+  withTheReader Context ({ · with inheritEnv := true }) z
+
+def newGoalState (goalState : GoalState) : MainM Nat := do
   let state ← get
   let stateId := state.nextId
   set { state with
@@ -27,42 +42,168 @@ def newGoalState (goalState: GoalState) : MainM Nat := do
   }
   return stateId
 
+def runCoreM { α } (coreM : CoreM α) : EMainM α := do
+  let scope := (← get).scope
+  let options := (← get).options
+  let cancelTk? ← match options.timeout with
+    | 0 => pure .none
+    | _ => .some <$> IO.CancelToken.new
+  let coreCtx : Core.Context := {
+    (← read).coreContext with
+    currNamespace      := scope.currNamespace,
+    openDecls          := scope.openDecls,
+    options            := scope.opts,
+    initHeartbeats     :=  ← IO.getNumHeartbeats,
+    cancelTk?,
+  }
+  let coreState : Core.State := {
+    env := (← get).env
+  }
+  -- Remap the coreM to capture every exception
+  let coreM' : CoreM _ :=
+    try
+      Except.ok <$> coreM
+    catch ex =>
+      let desc ← ex.toMessageData.toString
+      return Except.error ({ error := "exception", desc } : Protocol.InteractionError)
+    finally
+      for {msg, ..} in (← getTraceState).traces do
+        IO.eprintln (← msg.format.toIO)
+      resetTraceState
+  if let .some token := cancelTk? then
+    runCancelTokenWithTimeout token (timeout := .ofBitVec options.timeout)
+  let (result, state') ← match ← (coreM'.run coreCtx coreState).toIO' with
+    | Except.error (Exception.error _ msg)   => Protocol.throw $ { error := "core", desc := ← msg.toString }
+    | Except.error (Exception.internal id _) => Protocol.throw $ { error := "internal", desc := (← id.getName).toString }
+    | Except.ok a                            => pure a
+  if (← read).inheritEnv && result matches .ok _ then
+    setEnv state'.env
+  liftExcept result
 
-def runMetaInMainM { α } (metaM: MetaM α): MainM α :=
-  metaM.run'
-def runTermElabInMainM { α } (termElabM: Elab.TermElabM α) : MainM α :=
-  termElabM.run' (ctx := defaultElabContext) |>.run'
+def runCoreM' { α } (coreM : Protocol.FallibleT CoreM α) : EMainM α := do
+  liftExcept $ ← runCoreM coreM.run
+
+def liftMetaM { α } (metaM : MetaM α): EMainM α :=
+  runCoreM metaM.run'
+def liftTermElabM { α } (termElabM : Elab.TermElabM α) (levelNames : List Name := [])
+    : EMainM α := do
+  let scope := (← get).scope
+  let context := {
+    errToSorry := false,
+    isNoncomputableSection := scope.isNoncomputable,
+  }
+  let state := {
+    levelNames := scope.levelNames ++ levelNames,
+  }
+  runCoreM $ termElabM.run' context state |>.run'
+
+section Goal
+
+def goal_tactic (args: Protocol.GoalTactic): EMainM Protocol.GoalTacticResult := do
+  let state ← getMainState
+  let .some goalState := state.goalStates[args.stateId]? |
+    Protocol.throw $ Protocol.errorIndex s!"Invalid state index {args.stateId}"
+  let unshielded := args.autoResume?.getD state.options.automaticMode
+  let site ← match args.goalId?, unshielded with
+    | .some goalId, true => do
+      let .some goal := goalState.goals[goalId]? |
+        Protocol.throw $ Protocol.errorIndex s!"Invalid goal index {goalId}"
+      pure (.prefer goal)
+    | .some goalId, false => do
+      let .some goal := goalState.goals[goalId]? |
+        Protocol.throw $ Protocol.errorIndex s!"Invalid goal index {goalId}"
+      pure (.focus goal)
+    | .none, true => pure .unfocus
+    | .none, false => do
+      let .some goal := goalState.mainGoal? |
+        Protocol.throw $ Protocol.errorIndex s!"No goals to be solved"
+      pure (.focus goal)
+  let nextGoalState?: Except _ TacticResult ← liftTermElabM do
+    -- NOTE: Should probably use a macro to handle this...
+    match args.tactic?, args.mode?, args.expr?, args.have?, args.let?, args.draft?  with
+    | .some tactic, .none, .none, .none, .none, .none => do
+      pure $ Except.ok $ ← goalState.tryTactic site tactic
+    | .none, .some mode, .none, .none, .none, .none => match mode with
+      | "tactic" => do -- Exit from the current fragment
+        pure $ Except.ok $ ← goalState.fragmentExit site
+      | "conv" => do
+        pure $ Except.ok $ ← goalState.convEnter site
+      | "calc" => do
+        pure $ Except.ok $ ← goalState.calcEnter site
+      | _ => pure $ .error $ Protocol.errorOperation s!"Invalid mode {mode}"
+    | .none, .none, .some expr, .none, .none, .none => do
+      pure $ Except.ok $ ← goalState.tryAssign site expr
+    | .none, .none, .none, .some type, .none, .none => do
+      let binderName := args.binderName?.getD ""
+      pure $ Except.ok $ ← goalState.tryHave site binderName type
+    | .none, .none, .none, .none, .some type, .none => do
+      let binderName := args.binderName?.getD ""
+      pure $ Except.ok $ ← goalState.tryLet site binderName type
+    | .none, .none, .none, .none, .none, .some draft => do
+      pure $ Except.ok $ ← goalState.tryDraft site draft
+    | _, _, _, _, _, _ =>
+      pure $ .error $ Protocol.errorOperation
+        "Exactly one of {tactic, mode, expr, have, let, draft} must be supplied"
+  match nextGoalState? with
+  | .error error => Protocol.throw error
+  | .ok (.success nextGoalState messages) => do
+    let env ← getEnv
+    let nextStateId ← newGoalState nextGoalState
+    let parentExprs := nextGoalState.parentExprs
+    let hasSorry := parentExprs.any λ
+      | .ok e => e.hasSorry
+      | .error _ => false
+    let hasUnsafe := parentExprs.any λ
+      | .ok e => env.hasUnsafe e
+      | .error _ => false
+    let goals ← runCoreM $ nextGoalState.serializeGoals (options := state.options) |>.run'
+    let messages ← messages.mapM (·.serialize)
+    return {
+      nextStateId? := .some nextStateId,
+      goals? := .some goals,
+      messages? := .some messages,
+      hasSorry,
+      hasUnsafe,
+    }
+  | .ok (.parseError message) =>
+    return { messages? := .none, parseError? := .some message }
+  | .ok (.invalidAction message) =>
+    Protocol.throw $ errorI "invalid" message
+  | .ok (.failure messages) =>
+    let messages ← messages.mapM (·.serialize)
+    return { messages? := .some messages }
+
+end Goal
 
 section Frontend
 
 structure CompilationUnit where
-  -- Should be the penultimate environment, but this is ok
+  -- Environment immediately before the unit
   env : Environment
   boundary : Nat × Nat
   invocations : List Protocol.InvokedTactic
   sorrys : List Frontend.InfoWithContext
-  messages : Array String
+  messages : Array SerialMessage
   newConstants : List Name
 
-def frontend_process_inner (args: Protocol.FrontendProcess): MainM (CR Protocol.FrontendProcessResult) := do
-  let options := (← get).options
+def frontend_process (args: Protocol.FrontendProcess): EMainM Protocol.FrontendProcessResult := do
+  let options := (← getMainState).options
   let (fileName, file) ← match args.fileName?, args.file? with
     | .some fileName, .none => do
       let file ← IO.FS.readFile fileName
       pure (fileName, file)
     | .none, .some file =>
       pure ("<anonymous>", file)
-    | _, _ => return .error <| errorI "arguments" "Exactly one of {fileName, file} must be supplied"
-  let env?: Option Environment ← if args.fileName?.isSome then
+    | _, _ => Protocol.throw $ errorI "arguments" "Exactly one of {fileName, file} must be supplied"
+  let env?: Option Environment ← if args.readHeader then
       pure .none
     else do
-      let env ← getEnv
-      pure <| .some env
+      .some <$> getEnv
   let (context, state) ← do Frontend.createContextStateFromFile file fileName env? {}
-  let frontendM: Elab.Frontend.FrontendM (List CompilationUnit) :=
+  let frontendM: Frontend.FrontendM (List CompilationUnit) :=
     Frontend.mapCompilationSteps λ step => do
     let boundary := (step.src.startPos.byteIdx, step.src.stopPos.byteIdx)
-    let invocations: Option (List Protocol.InvokedTactic) ← if args.invocations then
+    let invocations: Option (List Protocol.InvokedTactic) ← if args.invocations?.isSome then
         Frontend.collectTacticsFromCompilationStep step
       else
         pure []
@@ -70,7 +211,7 @@ def frontend_process_inner (args: Protocol.FrontendProcess): MainM (CR Protocol.
         Frontend.collectSorrys step (options := { collectTypeErrors := args.typeErrorsAsGoals })
       else
         pure []
-    let messages ← step.messageStrings
+    let messages ← step.msgs.toArray.mapM (·.serialize)
     let newConstants ← if args.newConstants then
         Frontend.collectNewDefinedConstants step
       else
@@ -83,113 +224,130 @@ def frontend_process_inner (args: Protocol.FrontendProcess): MainM (CR Protocol.
       messages,
       newConstants
     }
-  let li ← frontendM.run context |>.run' state
+  let cancelTk? ← match (← get).options.timeout with
+    | 0 => pure .none
+    | timeout => .some <$> spawnCancelToken (timeout := .ofBitVec timeout)
+  let (li, state') ← frontendM.run { cancelTk? } |>.run context |>.run state
+  if args.inheritEnv then
+    setEnv state'.commandState.env
+    if let .some scope := state'.commandState.scopes.head? then
+      -- modify the scope
+      set { ← getMainState with scope }
+  if let .some fileName := args.invocations? then
+    let units := li.map λ unit => { invocations? := .some unit.invocations }
+    let data : Protocol.FrontendData := { units }
+    IO.FS.writeFile fileName (toJson data |>.compress)
   let units ← li.mapM λ step => withEnv step.env do
     let newConstants? := if args.newConstants then
         .some $ step.newConstants.toArray.map λ name => name.toString
       else
         .none
-    let (goalStateId?, goals?, goalSrcBoundaries?) ← if step.sorrys.isEmpty then do
+    let (goalStateId?, goals?, goalSrcBoundaries?) ← if step.sorrys.isEmpty then
         pure (.none, .none, .none)
       else do
-        let { state, srcBoundaries } ← runMetaInMainM $ Frontend.sorrysToGoalState step.sorrys
+        let ({ state, srcBoundaries }, goals) ← liftMetaM do
+          let result@{state, .. } ← Frontend.sorrysToGoalState step.sorrys
+          let goals ← goalSerialize state options
+          pure (result, goals)
         let stateId ← newGoalState state
-        let goals ← goalSerialize state options
         let srcBoundaries := srcBoundaries.toArray.map (λ (b, e) => (b.byteIdx, e.byteIdx))
         pure (.some stateId, .some goals, .some srcBoundaries)
-    let invocations? := if args.invocations then .some step.invocations else .none
+    let nInvocations? := if args.invocations?.isSome then .some step.invocations.length else .none
     return {
       boundary := step.boundary,
       messages := step.messages,
-      invocations?,
+      nInvocations?,
       goalStateId?,
       goals?,
       goalSrcBoundaries?,
       newConstants?,
     }
-  return .ok { units }
+  return { units }
 
 end Frontend
 
 /-- Main loop command of the REPL -/
 def execute (command: Protocol.Command): MainM Json := do
-  let run { α β: Type } [FromJson α] [ToJson β] (comm: α → MainM (CR β)): MainM Json :=
-    match fromJson? command.payload with
-    | .ok args => do
-      match (← comm args) with
-      | .ok result =>  return toJson result
-      | .error ierror => return toJson ierror
-    | .error error => return toJson $ errorCommand s!"Unable to parse json: {error}"
-  try
-    match command.cmd with
-    | "reset"         => run reset
-    | "stat"          => run stat
-    | "expr.echo"     => run expr_echo
-    | "env.describe"  => run env_describe
-    | "env.module_read" => run env_module_read
-    | "env.catalog"   => run env_catalog
-    | "env.inspect"   => run env_inspect
-    | "env.add"       => run env_add
-    | "env.save"      => run env_save
-    | "env.load"      => run env_load
-    | "options.set"   => run options_set
-    | "options.print" => run options_print
-    | "goal.start"    => run goal_start
-    | "goal.tactic"   => run goal_tactic
-    | "goal.continue" => run goal_continue
-    | "goal.delete"   => run goal_delete
-    | "goal.print"    => run goal_print
-    | "goal.save"     => run goal_save
-    | "goal.load"     => run goal_load
-    | "frontend.process"  => run frontend_process
-    | cmd =>
-      let error: Protocol.InteractionError :=
-        errorCommand s!"Unknown command {cmd}"
+  let run { α β: Type } [FromJson α] [ToJson β] (comm: α → EMainM β): MainM Json :=
+    try
+      match fromJson? command.payload with
+      | .ok args => do
+        let (msg, result) ← IO.FS.withIsolatedStreams (isolateStderr := false) $ comm args
+        if !msg.isEmpty then
+          IO.eprint s!"stdout: {msg}"
+        match result with
+        | .ok result =>  return toJson result
+        | .error ierror => return toJson ierror
+      | .error error => return toJson $ errorCommand s!"Unable to parse json: {error}"
+    catch ex : IO.Error =>
+      let error : Protocol.InteractionError := { error := "io", desc := ex.toString }
       return toJson error
-  catch ex => do
-    let error ← ex.toMessageData.toString
-    return toJson $ errorIO error
+  match command.cmd with
+  | "reset"         => run reset
+  | "stat"          => run stat
+  | "expr.echo"     => run expr_echo
+  | "env.describe"  => run env_describe
+  | "env.module_read" => run env_module_read
+  | "env.catalog"   => run env_catalog
+  | "env.inspect"   => run env_inspect
+  | "env.add"       => run env_add
+  | "env.save"      => run env_save
+  | "env.load"      => run env_load
+  | "options.set"   => run options_set
+  | "options.print" => run options_print
+  | "goal.start"    => run goal_start
+  | "goal.tactic"   => run goal_tactic
+  | "goal.continue" => run goal_continue
+  | "goal.delete"   => run goal_delete
+  | "goal.print"    => run goal_print
+  | "goal.save"     => run goal_save
+  | "goal.load"     => run goal_load
+  | "frontend.process" => run frontend_process
+  | cmd =>
+    let error: Protocol.InteractionError :=
+      errorCommand s!"Unknown command {cmd}"
+    return toJson error
   where
   errorCommand := errorI "command"
-  errorIndex := errorI "index"
   errorIO := errorI "io"
   -- Command Functions
-  reset (_: Protocol.Reset): MainM (CR Protocol.StatResult) := do
-    let state ← get
+  reset (_: Protocol.Reset): EMainM Protocol.StatResult := do
+    let state ← getMainState
     let nGoals := state.goalStates.size
-    set { state with nextId := 0, goalStates := .empty }
-    Core.resetMessageLog
-    return .ok { nGoals }
-  stat (_: Protocol.Stat): MainM (CR Protocol.StatResult) := do
-    let state ← get
+    set { state with nextId := 0, goalStates := .emptyWithCapacity }
+    return { nGoals }
+  stat (_: Protocol.Stat): EMainM Protocol.StatResult := do
+    let state ← getMainState
     let nGoals := state.goalStates.size
-    return .ok { nGoals }
-  env_describe (args: Protocol.EnvDescribe): MainM (CR Protocol.EnvDescribeResult) := do
-    let result ← Environment.describe args
-    return .ok result
-  env_module_read (args: Protocol.EnvModuleRead): MainM (CR Protocol.EnvModuleReadResult) := do
-    Environment.moduleRead args
-  env_catalog (args: Protocol.EnvCatalog): MainM (CR Protocol.EnvCatalogResult) := do
-    let result ← Environment.catalog args
-    return .ok result
-  env_inspect (args: Protocol.EnvInspect): MainM (CR Protocol.EnvInspectResult) := do
-    let state ← get
-    Environment.inspect args state.options
-  env_add (args: Protocol.EnvAdd): MainM (CR Protocol.EnvAddResult) := do
-    Environment.addDecl args
-  env_save (args: Protocol.EnvSaveLoad): MainM (CR Protocol.EnvSaveLoadResult) := do
+    return { nGoals }
+  env_describe (args: Protocol.EnvDescribe): EMainM Protocol.EnvDescribeResult := do
+    let result ← runCoreM $ Environment.describe args
+    return result
+  env_module_read (args: Protocol.EnvModuleRead): EMainM Protocol.EnvModuleReadResult := do
+    runCoreM $ Environment.moduleRead args
+  env_catalog (args: Protocol.EnvCatalog): EMainM Protocol.EnvCatalogResult := do
+    let result ← runCoreM $ Environment.catalog args
+    return result
+  env_inspect (args: Protocol.EnvInspect): EMainM Protocol.EnvInspectResult := do
+    let state ← getMainState
+    runCoreM' $ Environment.inspect args state.options
+  env_add (args: Protocol.EnvAdd): EMainM Protocol.EnvAddResult := withInheritEnv do
+    runCoreM' $ Environment.addDecl args.name (args.levels?.getD #[]) args.type? args.value args.isTheorem
+  env_save (args: Protocol.EnvSaveLoad): EMainM Protocol.EnvSaveLoadResult := do
     let env ← MonadEnv.getEnv
     environmentPickle env args.path
-    return .ok {}
-  env_load (args: Protocol.EnvSaveLoad): MainM (CR Protocol.EnvSaveLoadResult) := do
+    return {}
+  env_load (args: Protocol.EnvSaveLoad): EMainM Protocol.EnvSaveLoadResult := do
     let (env, _) ← environmentUnpickle args.path
     setEnv env
-    return .ok {}
-  expr_echo (args: Protocol.ExprEcho): MainM (CR Protocol.ExprEchoResult) := do
-    let state ← get
-    exprEcho args.expr (expectedType? := args.type?) (levels := args.levels.getD #[]) (options := state.options)
-  options_set (args: Protocol.OptionsSet): MainM (CR Protocol.OptionsSetResult) := do
-    let state ← get
+    return {}
+  expr_echo (args: Protocol.ExprEcho): EMainM Protocol.ExprEchoResult := do
+    let state ← getMainState
+    let levelNames := (args.levels?.getD #[]).toList.map (·.toName)
+    liftExcept $ ← liftTermElabM (levelNames := levelNames) do
+      (exprEcho args.expr (expectedType? := args.type?) (options := state.options)).run
+  options_set (args: Protocol.OptionsSet): EMainM Protocol.OptionsSetResult := do
+    let state ← getMainState
     let options := state.options
     set { state with
       options := {
@@ -202,136 +360,76 @@ def execute (command: Protocol.Command): MainM Json := do
         printAuxDecls := args.printAuxDecls?.getD options.printAuxDecls,
         printImplementationDetailHyps := args.printImplementationDetailHyps?.getD options.printImplementationDetailHyps
         automaticMode := args.automaticMode?.getD options.automaticMode,
+        timeout := args.timeout?.getD options.timeout,
       }
     }
-    return .ok {  }
-  options_print (_: Protocol.OptionsPrint): MainM (CR Protocol.Options) := do
-    return .ok (← get).options
-  goal_start (args: Protocol.GoalStart): MainM (CR Protocol.GoalStartResult) := do
-    let env ← MonadEnv.getEnv
-    let expr?: Except _ GoalState ← runTermElabInMainM (match args.expr, args.copyFrom with
-      | .some expr, .none => goalStartExpr expr (args.levels.getD #[])
-      | .none, .some copyFrom =>
-        (match env.find? <| copyFrom.toName with
-        | .none => return .error <| errorIndex s!"Symbol not found: {copyFrom}"
+    return {  }
+  options_print (_: Protocol.OptionsPrint): EMainM Protocol.Options := do
+    return (← getMainState).options
+  goal_start (args: Protocol.GoalStart): EMainM Protocol.GoalStartResult := do
+    let levelNames := (args.levels?.getD #[]).toList.map (·.toName)
+    let expr?: Except _ GoalState ← liftTermElabM (levelNames := levelNames) do
+      match args.expr, args.copyFrom with
+      | .some expr, .none => goalStartExpr expr |>.run
+      | .none, .some copyFrom => do
+        (match (← getEnv).find? <| copyFrom.toName with
+        | .none => return .error <| Protocol.errorIndex s!"Symbol not found: {copyFrom}"
         | .some cInfo => return .ok (← GoalState.create cInfo.type))
       | _, _ =>
-        return .error <| errorI "arguments" "Exactly one of {expr, copyFrom} must be supplied")
+        return .error <| errorI "arguments" "Exactly one of {expr, copyFrom} must be supplied"
     match expr? with
-    | .error error => return .error error
+    | .error error => Protocol.throw error
     | .ok goalState =>
       let stateId ← newGoalState goalState
-      return .ok { stateId, root := goalState.root.name.toString }
-  goal_tactic (args: Protocol.GoalTactic): MainM (CR Protocol.GoalTacticResult) := do
-    let state ← get
-    let .some goalState := state.goalStates[args.stateId]? |
-      return .error $ errorIndex s!"Invalid state index {args.stateId}"
-    let .some goal := goalState.goals.get? args.goalId |
-      return .error $ errorIndex s!"Invalid goal index {args.goalId}"
-    let nextGoalState?: Except _ TacticResult ← runTermElabInMainM do
-      -- NOTE: Should probably use a macro to handle this...
-      match args.tactic?, args.expr?, args.have?, args.let?, args.calc?, args.conv?, args.draft?  with
-      | .some tactic, .none, .none, .none, .none, .none, .none => do
-        pure <| Except.ok <| ← goalState.tryTactic goal tactic
-      | .none, .some expr, .none, .none, .none, .none, .none => do
-        pure <| Except.ok <| ← goalState.tryAssign goal expr
-      | .none, .none, .some type, .none, .none, .none, .none => do
-        let binderName := args.binderName?.getD ""
-        pure <| Except.ok <| ← goalState.tryHave goal binderName type
-      | .none, .none, .none, .some type, .none, .none, .none => do
-        let binderName := args.binderName?.getD ""
-        pure <| Except.ok <| ← goalState.tryLet goal binderName type
-      | .none, .none, .none, .none, .some pred, .none, .none => do
-        pure <| Except.ok <| ← goalState.tryCalc goal pred
-      | .none, .none, .none, .none, .none, .some true, .none => do
-        pure <| Except.ok <| ← goalState.conv goal
-      | .none, .none, .none, .none, .none, .some false, .none => do
-        pure <| Except.ok <| ← goalState.convExit
-      | .none, .none, .none, .none, .none, .none, .some draft => do
-        pure <| Except.ok <| ← goalState.tryDraft goal draft
-      | _, _, _, _, _, _, _ =>
-        let error := errorI "arguments" "Exactly one of {tactic, expr, have, calc, conv} must be supplied"
-        pure $ Except.error $ error
-    match nextGoalState? with
-    | .error error => return .error error
-    | .ok (.success nextGoalState) => do
-      let nextGoalState ← match state.options.automaticMode, args.conv? with
-        | true, .none => do
-          let .ok result := nextGoalState.resume (nextGoalState.goals ++ goalState.goals) |
-            throwError "Resuming known goals"
-          pure result
-        | true, .some true => pure nextGoalState
-        | true, .some false => do
-          let .some (_, _, dormantGoals) := goalState.convMVar? |
-            throwError "If conv exit succeeded this should not fail"
-          let .ok result := nextGoalState.resume (nextGoalState.goals ++ dormantGoals) |
-            throwError "Resuming known goals"
-          pure result
-        | false, _ => pure nextGoalState
-      let nextStateId ← newGoalState nextGoalState
-      let goals ← nextGoalState.serializeGoals (parent := .some goalState) (options := state.options) |>.run'
-      return .ok {
-        nextStateId? := .some nextStateId,
-        goals? := .some goals,
-      }
-    | .ok (.parseError message) =>
-      return .ok { parseError? := .some message }
-    | .ok (.invalidAction message) =>
-      return .error $ errorI "invalid" message
-    | .ok (.failure messages) =>
-      return .ok { tacticErrors? := .some messages }
-  goal_continue (args: Protocol.GoalContinue): MainM (CR Protocol.GoalContinueResult) := do
-    let state ← get
+      return { stateId, root := goalState.root.name.toString }
+  goal_continue (args: Protocol.GoalContinue): EMainM Protocol.GoalContinueResult := do
+    let state ← getMainState
     let .some target := state.goalStates[args.target]? |
-      return .error $ errorIndex s!"Invalid state index {args.target}"
-    let nextState? ← match args.branch?, args.goals? with
+      Protocol.throw $ Protocol.errorIndex s!"Invalid state index {args.target}"
+    let nextGoalState? : GoalState  ← match args.branch?, args.goals? with
       | .some branchId, .none => do
         match state.goalStates[branchId]? with
-        | .none => return .error $ errorIndex s!"Invalid state index {branchId}"
+        | .none => Protocol.throw $ Protocol.errorIndex s!"Invalid state index {branchId}"
         | .some branch => pure $ target.continue branch
       | .none, .some goals =>
-        pure $ goalResume target goals
-      | _, _ => return .error <| errorI "arguments" "Exactly one of {branch, goals} must be supplied"
-    match nextState? with
-    | .error error => return .error <| errorI "structure" error
+        let goals := goals.toList.map (λ n => { name := n.toName })
+        pure $ target.resume goals
+      | _, _ => Protocol.throw $ errorI "arguments" "Exactly one of {branch, goals} must be supplied"
+    match nextGoalState? with
+    | .error error => Protocol.throw $ errorI "structure" error
     | .ok nextGoalState =>
       let nextStateId ← newGoalState nextGoalState
-      let goals ← goalSerialize nextGoalState (options := state.options)
-      return .ok {
+      let goals ← liftMetaM $ goalSerialize nextGoalState (options := state.options)
+      return {
         nextStateId,
         goals,
       }
-  goal_delete (args: Protocol.GoalDelete): MainM (CR Protocol.GoalDeleteResult) := do
-    let state ← get
+  goal_delete (args: Protocol.GoalDelete): EMainM Protocol.GoalDeleteResult := do
+    let state ← getMainState
     let goalStates := args.stateIds.foldl (λ map id => map.erase id) state.goalStates
     set { state with goalStates }
-    return .ok {}
-  goal_print (args: Protocol.GoalPrint): MainM (CR Protocol.GoalPrintResult) := do
-    let state ← get
+    return {}
+  goal_print (args: Protocol.GoalPrint): EMainM Protocol.GoalPrintResult := do
+    let state ← getMainState
     let .some goalState := state.goalStates[args.stateId]? |
-      return .error $ errorIndex s!"Invalid state index {args.stateId}"
-    let result ← runMetaInMainM <| goalPrint
+      Protocol.throw $ Protocol.errorIndex s!"Invalid state index {args.stateId}"
+    let result ← liftMetaM <| goalPrint
         goalState
         (rootExpr := args.rootExpr?.getD False)
-        (parentExpr := args.parentExpr?.getD False)
+        (parentExprs := args.parentExprs?.getD False)
         (goals := args.goals?.getD False)
         (extraMVars := args.extraMVars?.getD #[])
         (options := state.options)
-    return .ok result
-  goal_save (args: Protocol.GoalSave): MainM (CR Protocol.GoalSaveResult) := do
-    let state ← get
+    return result
+  goal_save (args: Protocol.GoalSave): EMainM Protocol.GoalSaveResult := do
+    let state ← getMainState
     let .some goalState := state.goalStates[args.id]? |
-      return .error $ errorIndex s!"Invalid state index {args.id}"
-    goalStatePickle goalState args.path
-    return .ok {}
-  goal_load (args: Protocol.GoalLoad): MainM (CR Protocol.GoalLoadResult) := do
-    let (goalState, _) ← goalStateUnpickle args.path (← MonadEnv.getEnv)
+      Protocol.throw $ Protocol.errorIndex s!"Invalid state index {args.id}"
+    goalStatePickle goalState args.path (background? := .some $ ← getEnv)
+    return {}
+  goal_load (args: Protocol.GoalLoad): EMainM Protocol.GoalLoadResult := do
+    let (goalState, _) ← goalStateUnpickle args.path (background? := .some $ ← getEnv)
     let id ← newGoalState goalState
-    return .ok { id }
-  frontend_process (args: Protocol.FrontendProcess): MainM (CR Protocol.FrontendProcessResult) := do
-    try
-      frontend_process_inner args
-    catch e =>
-      return .error $ errorI "frontend" (← e.toMessageData.toString)
+    return { id }
 
 end Pantograph.Repl

Test/Common.lean

@@ -67,14 +67,15 @@ protected def Goal.devolatilize (goal: Goal): Goal :=
 
 end Condensed
 
-def GoalState.get! (state: GoalState) (i: Nat): MVarId := state.goals.get! i
-def GoalState.tacticOn (state: GoalState) (goalId: Nat) (tactic: String) := state.tryTactic (state.goals.get! goalId) tactic
+def GoalState.get! (state: GoalState) (i: Nat): MVarId := state.goals[i]!
+def GoalState.tacticOn (state: GoalState) (goalId: Nat) (tactic: String) := state.tryTactic (state.get! goalId) tactic
+def GoalState.tacticOn' (state: GoalState) (goalId: Nat) (tactic: TSyntax `tactic) :=
+  state.tryTacticM (state.get! goalId) (Elab.Tactic.evalTactic tactic) true
 
 def TacticResult.toString : TacticResult → String
-  | .success state => s!".success ({state.goals.length} goals)"
+  | .success state _messages => s!".success ({state.goals.length} goals)"
   | .failure messages =>
-    let messages := "\n".intercalate messages.toList
-    s!".failure {messages}"
+    s!".failure ({messages.size} messages)"
   | .parseError error => s!".parseError {error}"
   | .invalidAction error => s!".invalidAction {error}"
 
@@ -94,12 +95,14 @@ def runCoreMSeq (env: Environment) (coreM: CoreM LSpec.TestSeq) (options: Array
   | .ok a            => return a
 def runMetaMSeq (env: Environment) (metaM: MetaM LSpec.TestSeq): IO LSpec.TestSeq :=
   runCoreMSeq env metaM.run'
-def runTermElabMInMeta { α } (termElabM: Lean.Elab.TermElabM α): Lean.MetaM α :=
+def runTermElabMInMeta { α } (termElabM: Elab.TermElabM α): MetaM α :=
   termElabM.run' (ctx := defaultElabContext)
+def runTermElabMInCore { α } (termElabM: Elab.TermElabM α): CoreM α :=
+  (runTermElabMInMeta termElabM).run'
 def runTermElabMSeq (env: Environment) (termElabM: Elab.TermElabM LSpec.TestSeq): IO LSpec.TestSeq :=
   runMetaMSeq env $ termElabM.run' (ctx := defaultElabContext)
 
-def exprToStr (e: Expr): Lean.MetaM String := toString <$> Meta.ppExpr e
+def exprToStr (e: Expr): MetaM String := toString <$> Meta.ppExpr e
 
 def strToTermSyntax (s: String): CoreM Syntax := do
   let .ok stx := Parser.runParserCategory
@@ -108,7 +111,7 @@ def strToTermSyntax (s: String): CoreM Syntax := do
     (input := s)
     (fileName := ← getFileName) | panic! s!"Failed to parse {s}"
   return stx
-def parseSentence (s: String): Elab.TermElabM Expr := do
+def parseSentence (s : String) (expectedType? : Option Expr := .none) : Elab.TermElabM Expr := do
   let stx ← match Parser.runParserCategory
     (env := ← MonadEnv.getEnv)
     (catName := `term)
@@ -116,7 +119,7 @@ def parseSentence (s: String): Elab.TermElabM Expr := do
     (fileName := ← getFileName) with
     | .ok syn => pure syn
     | .error error => throwError "Failed to parse: {error}"
-  Elab.Term.elabTerm (stx := stx) .none
+  Elab.Term.elabTerm (stx := stx) expectedType?
 
 def runTacticOnMVar (tacticM: Elab.Tactic.TacticM Unit) (goal: MVarId): Elab.TermElabM (List MVarId) := do
     let (_, newGoals) ← tacticM { elaborator := .anonymous } |>.run { goals := [goal] }
@@ -157,6 +160,13 @@ end Monadic
 def runTestTermElabM (env: Environment) (t: TestT Elab.TermElabM Unit):
   IO LSpec.TestSeq :=
   runTermElabMSeq env $ runTest t
+def transformTestT { α } { μ μ' : Type → Type }
+    [Monad μ] [Monad μ'] [MonadLiftT (ST IO.RealWorld) μ] [MonadLiftT (ST IO.RealWorld) μ']
+    (tr : {β : Type} → μ β  → μ' β) (m : TestT μ α) : TestT μ' α := do
+  let tests ← get
+  let (a, tests) ← tr (β := α × LSpec.TestSeq) (m.run tests)
+  set tests
+  return a
 
 def cdeclOf (userName: Name) (type: Expr): Condensed.LocalDecl :=
   { userName, type }
@@ -172,6 +182,29 @@ def buildGoal (nameType: List (String × String)) (target: String) (userName?: O
     })).toArray
   }
 
+namespace Tactic
+
+/-- Create an aux lemma and assigns it to `mvarId`, which is circuitous, but
+exercises the aux lemma generator. -/
+def assignWithAuxLemma (type : Expr) (value? : Option Expr := .none) : Elab.Tactic.TacticM Unit := do
+  let type ← instantiateMVars type
+  let value ← match value? with
+    | .some value => instantiateMVars value
+    | .none => Meta.mkSorry type (synthetic := false)
+  if type.hasExprMVar then
+    throwError "Type has expression mvar"
+  if value.hasExprMVar then
+    throwError "value has expression mvar"
+  let goal ← Elab.Tactic.getMainGoal
+  goal.withContext do
+  let name ← Meta.mkAuxLemma [] type value
+  unless ← Meta.isDefEq type (← goal.getType) do
+    throwError "Type provided is incorrect"
+  goal.assign (.const name [])
+  Elab.Tactic.pruneSolvedGoals
+
+end Tactic
+
 end Test
 
 end Pantograph

Test/Delate.lean

@@ -3,10 +3,9 @@ import Pantograph.Delate
 import Test.Common
 import Lean
 
-open Lean
-namespace Pantograph.Test.Delate
+open Lean Pantograph
 
-open Pantograph
+namespace Pantograph.Test.Delate
 
 deriving instance Repr, DecidableEq for Protocol.BoundExpression
 
@@ -49,10 +48,9 @@ def test_sexp_of_symbol (env: Environment): IO LSpec.TestSeq := do
 def test_sexp_of_elab (env: Environment): IO LSpec.TestSeq := do
   let entries: List (String × (List Name) × String) := [
     ("λ x: Nat × Bool => x.1", [], "(:lambda x ((:c Prod) (:c Nat) (:c Bool)) ((:c Prod.fst) (:c Nat) (:c Bool) 0))"),
-    ("λ x: Array Nat => x.data", [], "(:lambda x ((:c Array) (:c Nat)) ((:c Array.data) (:c Nat) 0))"),
     ("λ {α: Sort (u + 1)} => List α", [`u], "(:lambda α (:sort (+ u 1)) ((:c List) 0) :i)"),
     ("λ {α} => List α", [], "(:lambda α (:sort (+ (:mv _uniq.4) 1)) ((:c List) 0) :i)"),
-    ("(2: Nat) <= (5: Nat)", [], "((:c LE.le) (:mv _uniq.18) (:mv _uniq.19) ((:c OfNat.ofNat) (:mv _uniq.4) (:lit 2) (:mv _uniq.5)) ((:c OfNat.ofNat) (:mv _uniq.14) (:lit 5) (:mv _uniq.15)))"),
+    ("(2: Nat) <= (5: Nat)", [], "((:c LE.le) (:mv _uniq.20) (:mv _uniq.21) ((:c OfNat.ofNat) (:mv _uniq.4) (:lit 2) (:mv _uniq.5)) ((:c OfNat.ofNat) (:mv _uniq.15) (:lit 5) (:mv _uniq.16)))"),
   ]
   entries.foldlM (λ suites (source, levels, target) =>
     let termElabM := do
@@ -113,6 +111,13 @@ def test_projection_exists (env: Environment) : IO LSpec.TestSeq:= runTest do
   checkEq "numParams" numParams 2
   checkEq "numFields" numFields 2
 
+def test_matcher : TestT Elab.TermElabM Unit := do
+  let t ← parseSentence "Nat → Nat"
+  let e ← parseSentence "fun (n : Nat) => match n with | 0 => 0 | k => k" (.some t)
+  let .some _ ← Meta.matchMatcherApp? e.bindingBody! | fail "Must be a matcher app"
+  let e' ← instantiateAll e
+  checkTrue "ok" <| ← Meta.isTypeCorrect e'
+
 def suite (env: Environment): List (String × IO LSpec.TestSeq) :=
   [
     ("serializeName", do pure test_serializeName),
@@ -123,6 +128,7 @@ def suite (env: Environment): List (String × IO LSpec.TestSeq) :=
     ("Instance", test_instance env),
     ("Projection Prod", test_projection_prod env),
     ("Projection Exists", test_projection_exists env),
+    ("Matcher", runTestTermElabM env test_matcher),
   ]
 
 end Pantograph.Test.Delate

Test/Environment.lean

@@ -15,11 +15,7 @@ deriving instance DecidableEq, Repr for Protocol.RecursorRule
 deriving instance DecidableEq, Repr for Protocol.RecursorInfo
 deriving instance DecidableEq, Repr for Protocol.EnvInspectResult
 
-def test_catalog: IO LSpec.TestSeq := do
-  let env: Environment ← importModules
-    (imports := #[`Init])
-    (opts := {})
-    (trustLevel := 1)
+def test_catalog (env : Environment) : IO LSpec.TestSeq := do
   let inner: CoreM LSpec.TestSeq := do
     let catalogResult ← Environment.catalog {}
     let symbolsWithNum := env.constants.fold (init := #[]) (λ acc name info =>
@@ -34,7 +30,6 @@ def test_symbol_visibility: IO LSpec.TestSeq := do
   let entries: List (Name × Bool) := [
     ("Nat.add_comm".toName, false),
     ("foo.bla.Init.Data.List.Basic.2.1.Init.Lean.Expr._hyg.4".toName, true),
-    ("Init.Data.Nat.Basic._auxLemma.4".toName, true),
   ]
   let suite := entries.foldl (λ suites (symbol, target) =>
     let test := LSpec.check symbol.toString ((Environment.isNameInternal symbol) == target)
@@ -46,11 +41,7 @@ inductive ConstantCat where
   | ctor (info: Protocol.ConstructorInfo)
   | recursor (info: Protocol.RecursorInfo)
 
-def test_inspect: IO LSpec.TestSeq := do
-  let env: Environment ← importModules
-    (imports := #[`Init])
-    (opts := {})
-    (trustLevel := 1)
+def test_inspect (env : Environment) : IO LSpec.TestSeq := do
   let testCases: List (String × ConstantCat) := [
     ("Or", ConstantCat.induct {
       numParams := 2,
@@ -97,29 +88,29 @@ def test_inspect: IO LSpec.TestSeq := do
     ) LSpec.TestSeq.done
   runCoreMSeq env inner
 
-def test_symbol_location : TestT IO Unit := do
-  let env: Environment ← importModules
-    (imports := #[`Init])
-    (opts := {})
-    (trustLevel := 1)
+def test_symbol_location (env : Environment) : TestT IO Unit := do
   addTest $ ← runTestCoreM env do
-    let .ok result ← Environment.inspect { name := "Nat.le_of_succ_le", source? := .some true } (options := {}) | fail "Inspect failed"
+    let .ok result ← (Environment.inspect { name := "Nat.le_of_succ_le", source? := .some true } (options := {})).run | fail "Inspect failed"
     checkEq "module" result.module? <| .some "Init.Data.Nat.Basic"
 
     -- Extraction of source doesn't work for symbols in `Init` for some reason
     checkTrue "file" result.sourceUri?.isNone
-    checkEq "pos" (result.sourceStart?.map (·.column)) <| .some 0
-    checkEq "pos" (result.sourceEnd?.map (·.column)) <| .some 88
-    let .ok { imports, constNames, .. } ← Environment.moduleRead ⟨"Init.Data.Nat.Basic"⟩ | fail "Module read failed"
-    checkEq "imports" imports #["Init.SimpLemmas", "Init.Data.NeZero"]
+    checkEq "sourceStart" (result.sourceStart?.map (·.column)) <| .some 0
+    checkEq "sourceEnd" (result.sourceEnd?.map (·.column)) <| .some 88
+    let { imports, constNames, .. } ← Environment.moduleRead ⟨"Init.Data.Nat.Basic"⟩
+    checkEq "imports" imports #["Init.SimpLemmas", "Init.Data.NeZero", "Init.Grind.Tactics"]
     checkTrue "constNames" $ constNames.contains "Nat.succ_add"
 
-def suite: List (String × IO LSpec.TestSeq) :=
+def test_matcher (env : Environment) : TestT IO Unit := do
+  checkTrue "not matcher" $ ¬ Meta.isMatcherCore env `Nat.strongRecOn
+
+def suite (env : Environment) : List (String × IO LSpec.TestSeq) :=
   [
-    ("Catalog", test_catalog),
+    ("Catalog", test_catalog env),
     ("Symbol Visibility", test_symbol_visibility),
-    ("Inspect", test_inspect),
-    ("Symbol Location", runTest test_symbol_location),
+    ("Inspect", test_inspect env),
+    ("Symbol Location", runTest $ test_symbol_location env),
+    ("Matcher", runTest $ test_matcher env),
   ]
 
 end Pantograph.Test.Environment

Test/Frontend.lean

@@ -1,18 +1,40 @@
-import LSpec
 import Pantograph
 import Repl
 import Test.Common
 
+import LSpec
+
 open Lean Pantograph
 namespace Pantograph.Test.Frontend
 
+open Frontend
+
+def runFrontend { α } (source: String) (f : CompilationStep → FrontendM α) (timeout : UInt32 := 0): MetaM (List α) := do
+  let filename := "<anonymous>"
+  let (context, state) ← do createContextStateFromFile source filename (← getEnv) {}
+  let m := mapCompilationSteps f
+  let cancelTk? ← match timeout with
+    | 0 => pure .none
+    | timeout => .some <$> spawnCancelToken timeout
+  m.run { cancelTk? } |>.run context |>.run' state
+
+def test_open : TestT MetaM Unit := do
+  let sketch := "
+open Nat
+example : ∀ (n : Nat), n + 1 = Nat.succ n := by
+  intro
+  apply add_one
+  "
+  let errors ← runFrontend sketch λ step => step.msgs.mapM (·.toString)
+  checkEq "errors" errors [[], []]
+
 def collectSorrysFromSource (source: String) (options : Frontend.GoalCollectionOptions := {})
     : MetaM (List GoalState) := do
   let filename := "<anonymous>"
   let (context, state) ← do Frontend.createContextStateFromFile source filename (← getEnv) {}
   let m := Frontend.mapCompilationSteps λ step => do
     return (step.before, ← Frontend.collectSorrys step options)
-  let li ← m.run context |>.run' state
+  let li ← m.run {} |>.run context |>.run' state
   let goalStates ← li.filterMapM λ (env, sorrys) => withEnv env do
     if sorrys.isEmpty then
       return .none
@@ -160,6 +182,43 @@ example : ∀ (y: Nat), ∃ (x: Nat), y + 1 = x := by
     }
   ])
 
+def test_sorry_with_local_instance : TestT MetaM Unit := do
+  let sketch := "
+def test (α : Type) [s : Inhabited α] : α := @Inhabited.default α s
+example (α : Type) [Inhabited α] : α := sorry
+  "
+  let goalStates ← (collectSorrysFromSource sketch).run' {}
+  let [goalState] := goalStates | panic! s!"Incorrect number of states: {goalStates.length}"
+  let result ← runTermElabMInMeta $ goalState.tryTactic .unfocus "exact test α"
+  checkTrue "success" $ result matches .success ..
+  match result with
+  | .success .. => return ()
+  | .failure messages =>
+    let messages ← messages.mapM (·.toString)
+    fail s!"Could not execute tactic {messages}"
+  | .parseError e =>
+    fail s!"Parse error: {e}"
+  | .invalidAction e =>
+    fail s!"Invalid action: {e}"
+
+def test_sorry_circular : TestT MetaM Unit := do
+  let sketch := "
+theorem test (p q : Prop) (hp : p) (hq : q) : p ∧ q ∧ p := by sorry
+  "
+  let goalStates ← (collectSorrysFromSource sketch).run' {}
+  let [goalState] := goalStates | panic! s!"Incorrect number of states: {goalStates.length}"
+  let result ← runTermElabMInMeta $ goalState.tryTactic .unfocus "exact test α"
+  checkTrue "failure" $ result matches .failure ..
+  match result with
+  | .success .. =>
+    fail s!"This should not succeed"
+  | .failure .. =>
+    return ()
+  | .parseError e =>
+    fail s!"Parse error: {e}"
+  | .invalidAction e =>
+    fail s!"Invalid action: {e}"
+
 def test_environment_capture: TestT MetaM Unit := do
   let sketch := "
 def mystery (n: Nat) := n + 1
@@ -210,7 +269,7 @@ def collectNewConstants (source: String) : MetaM (List (List Name)) := do
   let (context, state) ← do Frontend.createContextStateFromFile source filename (← getEnv) {}
   let m := Frontend.mapCompilationSteps λ step => do
     Frontend.collectNewDefinedConstants step
-  m.run context |>.run' state
+  m.run {} |>.run context |>.run' state
 
 def test_collect_one_constant : TestT MetaM Unit := do
   let input := "
@@ -233,10 +292,13 @@ theorem mystery [SizeOf α] (as : List α) (i : Fin as.length) : sizeOf (as.get
 
 def suite (env : Environment): List (String × IO LSpec.TestSeq) :=
   let tests := [
+    ("open", test_open),
     ("multiple_sorrys_in_proof", test_multiple_sorrys_in_proof),
-    ("sorry_in_middle", test_sorry_in_middle),
-    ("sorry_in_induction", test_sorry_in_induction),
-    ("sorry_in_coupled", test_sorry_in_coupled),
+    ("sorry in middle", test_sorry_in_middle),
+    ("sorry in induction", test_sorry_in_induction),
+    ("sorry in coupled", test_sorry_in_coupled),
+    ("sorry with local instances", test_sorry_with_local_instance),
+    ("sorry circular", test_sorry_circular),
     ("environment_capture", test_environment_capture),
     ("capture_type_mismatch", test_capture_type_mismatch),
     --("capture_type_mismatch_in_binder", test_capture_type_mismatch_in_binder),

Test/Integration.lean

@@ -8,76 +8,124 @@ import Test.Common
 namespace Pantograph.Test.Integration
 open Pantograph.Repl
 
-def step { α } [Lean.ToJson α] (cmd: String) (payload: List (String × Lean.Json))
-    (expected: α) (name? : Option String := .none): MainM LSpec.TestSeq := do
-  let payload := Lean.Json.mkObj payload
+deriving instance Lean.ToJson for Protocol.EnvInspect
+deriving instance Lean.ToJson for Protocol.EnvAdd
+deriving instance Lean.ToJson for Protocol.ExprEcho
+deriving instance Lean.ToJson for Protocol.OptionsSet
+deriving instance Lean.ToJson for Protocol.OptionsPrint
+deriving instance Lean.ToJson for Protocol.GoalStart
+deriving instance Lean.ToJson for Protocol.GoalPrint
+deriving instance Lean.ToJson for Protocol.GoalTactic
+deriving instance Lean.ToJson for Protocol.FrontendProcess
+deriving instance Lean.ToJson for Protocol.FrontendDataUnit
+deriving instance Lean.ToJson for Protocol.FrontendData
+
+abbrev TestM α := TestT MainM α
+abbrev Test := TestM Unit
+
+def getFileName : TestM String := do
+  return (← read).coreContext.fileName
+
+def step { α β } [Lean.ToJson α] [Lean.ToJson β] (cmd: String) (payload: α)
+    (expected: β) (name? : Option String := .none) : TestM Unit := do
+  let payload := Lean.toJson payload
   let name := name?.getD s!"{cmd} {payload.compress}"
   let result ← Repl.execute { cmd, payload }
-  return LSpec.test name (toString result = toString (Lean.toJson expected))
+  checkEq name result.compress (Lean.toJson expected).compress
+def stepFile { α } [Lean.ToJson α] (name : String) (path : String)
+    (expected : α) : TestM Unit := do
+  let content ← IO.FS.readFile path
+  let payload? : Except String Lean.Json := Lean.Json.parse content
+  match payload? with
+  | .ok payload =>
+    let expected := Lean.toJson expected
+    checkEq name payload.compress expected.compress
+  | .error e => fail s!"{name} {e}"
 
-abbrev Test := List (MainM LSpec.TestSeq)
+def test_expr_echo : Test :=
+  step "expr.echo"
+   ({ expr := "λ {α : Sort (u + 1)} => List α", levels? := .some #["u"]}: Protocol.ExprEcho)
+   ({
+     type := { pp? := .some "{α : Type u} → Type u" },
+     expr := { pp? := .some "fun {α} => List α" }
+   }: Protocol.ExprEchoResult)
 
-def test_elab : Test :=
-  [
-    step "expr.echo"
-      [("expr", .str "λ {α : Sort (u + 1)} => List α"), ("levels", .arr #["u"])]
-     (Lean.toJson ({
-       type := { pp? := .some "{α : Type u} → Type u" },
-       expr := { pp? := .some "fun {α} => List α" }
-     }: Protocol.ExprEchoResult)),
-  ]
-
-def test_option_modify : Test :=
+def test_option_modify : Test := do
   let pp? := Option.some "∀ (n : Nat), n + 1 = n.succ"
   let sexp? := Option.some "(:forall n (:c Nat) ((:c Eq) (:c Nat) ((:c HAdd.hAdd) (:c Nat) (:c Nat) (:c Nat) ((:c instHAdd) (:c Nat) (:c instAddNat)) 0 ((:c OfNat.ofNat) (:c Nat) (:lit 1) ((:c instOfNatNat) (:lit 1)))) ((:c Nat.succ) 0)))"
   let module? := Option.some "Init.Data.Nat.Basic"
   let options: Protocol.Options := {}
-  [
-    step "env.inspect" [("name", .str "Nat.add_one")]
-     ({ type := { pp? }, module? }: Protocol.EnvInspectResult),
-    step "options.set" [("printExprAST", .bool true)]
-     ({ }: Protocol.OptionsSetResult),
-    step "env.inspect" [("name", .str "Nat.add_one")]
-     ({ type := { pp?, sexp? }, module? }: Protocol.EnvInspectResult),
-    step "options.print" []
-     ({ options with printExprAST := true }: Protocol.Options),
-  ]
-def test_malformed_command : Test :=
+  step "env.inspect" ({ name := "Nat.add_one" } : Protocol.EnvInspect)
+   ({ type := { pp? }, module? }: Protocol.EnvInspectResult)
+  step "options.set" ({ printExprAST? := .some true } : Protocol.OptionsSet)
+   ({ }: Protocol.OptionsSetResult)
+  step "env.inspect" ({ name := "Nat.add_one" } : Protocol.EnvInspect)
+   ({ type := { pp?, sexp? }, module? }: Protocol.EnvInspectResult)
+  step "options.print" ({} : Protocol.OptionsPrint)
+   ({ options with printExprAST := true }: Protocol.Options)
+
+def test_malformed_command : Test := do
   let invalid := "invalid"
-  [
-    step invalid [("name", .str "Nat.add_one")]
-     ({ error := "command", desc := s!"Unknown command {invalid}" }: Protocol.InteractionError)
-     (name? := .some "Invalid Command"),
-    step "expr.echo" [(invalid, .str "Random garbage data")]
-     ({ error := "command", desc := s!"Unable to parse json: Pantograph.Protocol.ExprEcho.expr: String expected" }:
-      Protocol.InteractionError)
-    (name? := .some "JSON Deserialization")
-  ]
-def test_tactic : Test :=
+  step invalid ({ name := "Nat.add_one" }: Protocol.EnvInspect)
+   ({ error := "command", desc := s!"Unknown command {invalid}" }: Protocol.InteractionError)
+   (name? := .some "Invalid Command")
+  step "expr.echo" (Lean.Json.mkObj [(invalid, .str "Random garbage data")])
+   ({ error := "command", desc := s!"Unable to parse json: Pantograph.Protocol.ExprEcho.expr: String expected" }:
+    Protocol.InteractionError) (name? := .some "JSON Deserialization")
+def test_tactic : Test := do
+  let varX := { name := "_uniq.10", userName := "x", type? := .some { pp? := .some "Prop" }}
   let goal1: Protocol.Goal := {
     name := "_uniq.11",
     target := { pp? := .some "∀ (q : Prop), x ∨ q → q ∨ x" },
-    vars := #[{ name := "_uniq.10", userName := "x", type? := .some { pp? := .some "Prop" }}],
+    vars := #[varX],
   }
   let goal2: Protocol.Goal := {
-    name := "_uniq.17",
+    name := "_uniq.14",
     target := { pp? := .some "x ∨ y → y ∨ x" },
     vars := #[
-      { name := "_uniq.10", userName := "x", type? := .some { pp? := .some "Prop" }},
-      { name := "_uniq.16", userName := "y", type? := .some { pp? := .some "Prop" }}
+      varX,
+      { name := "_uniq.13", userName := "y", type? := .some { pp? := .some "Prop" }}
     ],
   }
-  [
-    step "goal.start" [("expr", .str "∀ (p q: Prop), p ∨ q → q ∨ p")]
-     ({ stateId := 0, root := "_uniq.9" }: Protocol.GoalStartResult),
-    step "goal.tactic" [("stateId", .num 0), ("goalId", .num 0), ("tactic", .str "intro x")]
-     ({ nextStateId? := .some 1, goals? := #[goal1], }: Protocol.GoalTacticResult),
-    step "goal.print" [("stateId", .num 1), ("parentExpr", .bool true), ("rootExpr", .bool true)]
-     ({ parent? := .some { pp? := .some "fun x => ?m.11" }, }: Protocol.GoalPrintResult),
-    step "goal.tactic" [("stateId", .num 1), ("goalId", .num 0), ("tactic", .str "intro y")]
-     ({ nextStateId? := .some 2, goals? := #[goal2], }: Protocol.GoalTacticResult),
-  ]
-def test_automatic_mode (automatic: Bool): Test :=
+  step "goal.start" ({ expr := "∀ (p q: Prop), p ∨ q → q ∨ p" }: Protocol.GoalStart)
+   ({ stateId := 0, root := "_uniq.9" }: Protocol.GoalStartResult)
+  step "goal.tactic" ({ stateId := 0, tactic? := .some "intro x" }: Protocol.GoalTactic)
+   ({ nextStateId? := .some 1, goals? := #[goal1], }: Protocol.GoalTacticResult)
+  step "goal.print" ({ stateId := 1, parentExprs? := .some true, rootExpr? := .some true }: Protocol.GoalPrint)
+   ({
+     root? := .some { pp? := "fun x => ?m.11"},
+     parentExprs? := .some [.some { pp? := .some "fun x => ?m.11" }],
+   }: Protocol.GoalPrintResult)
+  step "goal.tactic" ({ stateId := 1, tactic? := .some "intro y" }: Protocol.GoalTactic)
+   ({ nextStateId? := .some 2, goals? := #[goal2], }: Protocol.GoalTacticResult)
+  step "goal.tactic" ({ stateId := 1, tactic? := .some "apply Nat.le_of_succ_le" }: Protocol.GoalTactic)
+    ({
+      messages? := .some #[{
+        fileName := ← getFileName,
+        kind := .anonymous,
+        pos := ⟨0, 0⟩,
+        data := "tactic 'apply' failed, could not unify the conclusion of `@Nat.le_of_succ_le`\n  ∀ {m : Nat}, Nat.succ ?n ≤ m → ?n ≤ m\nwith the goal\n  ∀ (q : Prop), x ∨ q → q ∨ x\n\nNote: The full type of `@Nat.le_of_succ_le` is\n  ∀ {n m : Nat}, n.succ ≤ m → n ≤ m\nx : Prop\n⊢ ∀ (q : Prop), x ∨ q → q ∨ x",
+      }]
+    }: Protocol.GoalTacticResult)
+  step "goal.tactic" ({ stateId := 0, tactic? := .some "sorry" }: Protocol.GoalTactic)
+   ({ nextStateId? := .some 3, goals? := .some #[], hasSorry := true }: Protocol.GoalTacticResult)
+example : (1 : Nat) + (2 * 3) = 1 + (4 - 3) + (6 - 4) + 3 := by
+  simp
+def test_tactic_timeout : Test := do
+  step "goal.start" ({ expr := "(1 : Nat) + (2 * 3) = 1 + (4 - 3) + (6 - 4) + 3" }: Protocol.GoalStart)
+   ({ stateId := 0, root := "_uniq.370" }: Protocol.GoalStartResult)
+  -- timeout of 10 milliseconds
+  step "options.set" ({ timeout? := .some 10 } : Protocol.OptionsSet)
+   ({ }: Protocol.OptionsSetResult)
+  step "goal.tactic" ({ stateId := 0, expr? := .some "by\nsleep 1000; simp" }: Protocol.GoalTactic)
+   ({ error := "internal", desc := "interrupt" }: Protocol.InteractionError)
+  -- ensure graceful recovery
+  step "options.set" ({ timeout? := .some 0 } : Protocol.OptionsSet)
+   ({ }: Protocol.OptionsSetResult)
+  step "goal.tactic" ({ stateId := 0, tactic? := .some "simp" }: Protocol.GoalTactic)
+   ({ nextStateId? := .some 1, goals? := .some #[], }: Protocol.GoalTacticResult)
+
+def test_automatic_mode (automatic: Bool): Test := do
   let varsPQ := #[
     { name := "_uniq.10", userName := "p", type? := .some { pp? := .some "Prop" }},
     { name := "_uniq.13", userName := "q", type? := .some { pp? := .some "Prop" }}
@@ -113,149 +161,267 @@ def test_automatic_mode (automatic: Bool): Test :=
       { name := "_uniq.49", userName := "h✝", type? := .some { pp? := .some "p" }, isInaccessible := true}
     ],
   }
-  [
-    step "options.set" [("automaticMode", .bool automatic)]
-     ({}: Protocol.OptionsSetResult),
-    step "goal.start" [("expr", .str "∀ (p q: Prop), p ∨ q → q ∨ p")]
-     ({ stateId := 0, root := "_uniq.9" }: Protocol.GoalStartResult),
-    step "goal.tactic" [("stateId", .num 0), ("goalId", .num 0), ("tactic", .str "intro p q h")]
-     ({ nextStateId? := .some 1, goals? := #[goal1], }: Protocol.GoalTacticResult),
-    step "goal.tactic" [("stateId", .num 1), ("goalId", .num 0), ("tactic", .str "cases h")]
-     ({ nextStateId? := .some 2, goals? := #[goal2l, goal2r], }: Protocol.GoalTacticResult),
-    let goals? := if automatic then #[goal3l, goal2r] else #[goal3l]
-    step "goal.tactic" [("stateId", .num 2), ("goalId", .num 0), ("tactic", .str "apply Or.inr")]
-     ({ nextStateId? := .some 3, goals?, }: Protocol.GoalTacticResult),
-  ]
+  step "options.set" ({automaticMode? := .some automatic}: Protocol.OptionsSet)
+   ({}: Protocol.OptionsSetResult)
+  step "goal.start" ({ expr := "∀ (p q: Prop), p ∨ q → q ∨ p"} : Protocol.GoalStart)
+   ({ stateId := 0, root := "_uniq.9" }: Protocol.GoalStartResult)
+  step "goal.tactic" ({ stateId := 0, tactic? := .some "intro p q h" }: Protocol.GoalTactic)
+   ({ nextStateId? := .some 1, goals? := #[goal1], }: Protocol.GoalTacticResult)
+  step "goal.tactic" ({ stateId := 1, tactic? := .some "cases h" }: Protocol.GoalTactic)
+   ({ nextStateId? := .some 2, goals? := #[goal2l, goal2r], }: Protocol.GoalTacticResult)
+  let goals? := if automatic then #[goal3l, goal2r] else #[goal3l]
+  step "goal.tactic" ({ stateId := 2, tactic? := .some "apply Or.inr" }: Protocol.GoalTactic)
+   ({ nextStateId? := .some 3, goals?, }: Protocol.GoalTacticResult)
 
-def test_env_add_inspect : Test :=
+def test_conv_calc : Test := do
+  step "options.set" ({automaticMode? := .some false}: Protocol.OptionsSet)
+   ({}: Protocol.OptionsSetResult)
+  step "goal.start" ({ expr := "∀ (a b: Nat), (b = 2) -> 1 + a + 1 = a + b"} : Protocol.GoalStart)
+   ({ stateId := 0, root := "_uniq.171" }: Protocol.GoalStartResult)
+  let vars := #[
+    { name := "_uniq.172", userName := "a", type? := .some { pp? := .some "Nat" }},
+    { name := "_uniq.175", userName := "b", type? := .some { pp? := .some "Nat" }},
+    { name := "_uniq.178", userName := "h", type? := .some { pp? := .some "b = 2" }},
+  ]
+  let goal : Protocol.Goal := {
+    vars,
+    name := "_uniq.179",
+    target := { pp? := "1 + a + 1 = a + b" },
+  }
+  step "goal.tactic" ({ stateId := 0, tactic? := .some "intro a b h" }: Protocol.GoalTactic)
+   ({ nextStateId? := .some 1, goals? := #[goal], }: Protocol.GoalTacticResult)
+  step "goal.tactic" ({ stateId := 1, mode? := .some "calc" }: Protocol.GoalTactic)
+   ({ nextStateId? := .some 2, goals? := #[{ goal with fragment := .calc }], }: Protocol.GoalTacticResult)
+  let goalCalc : Protocol.Goal := {
+    vars,
+    name := "_uniq.381",
+    userName? := .some "calc",
+    target := { pp? := "1 + a + 1 = a + 1 + 1" },
+  }
+  let goalMain : Protocol.Goal := {
+    vars,
+    name := "_uniq.400",
+    fragment := .calc,
+    target := { pp? := "a + 1 + 1 = a + b" },
+  }
+  step "goal.tactic" ({ stateId := 2, tactic? := .some "1 + a + 1 = a + 1 + 1" }: Protocol.GoalTactic)
+   ({ nextStateId? := .some 3, goals? := #[goalCalc, goalMain], }: Protocol.GoalTacticResult)
+  let goalConv : Protocol.Goal := {
+    goalCalc with
+    fragment := .conv,
+    userName? := .none,
+    name := "_uniq.468",
+  }
+  step "goal.tactic" ({ stateId := 3, mode? := .some "conv" }: Protocol.GoalTactic)
+   ({ nextStateId? := .some 4, goals? := #[goalConv], }: Protocol.GoalTacticResult)
+
+def test_env_add_inspect : Test := do
   let name1 := "Pantograph.mystery"
   let name2 := "Pantograph.mystery2"
-  [
-    step "env.add"
-      [
-        ("name", .str name1),
-        ("type", .str "Prop → Prop → Prop"),
-        ("value", .str "λ (a b: Prop) => Or a b"),
-        ("isTheorem", .bool false)
-      ]
-     ({}: Protocol.EnvAddResult),
-    step "env.inspect" [("name", .str name1)]
-     ({
-       value? := .some { pp? := .some "fun a b => a ∨ b" },
-       type := { pp? := .some "Prop → Prop → Prop" },
-     }:
-      Protocol.EnvInspectResult),
-    step "env.add"
-      [
-        ("name", .str name2),
-        ("type", .str "Nat → Int"),
-        ("value", .str "λ (a: Nat) => a + 1"),
-        ("isTheorem", .bool false)
-      ]
-     ({}: Protocol.EnvAddResult),
-    step "env.inspect" [("name", .str name2)]
-     ({
-       value? := .some { pp? := .some "fun a => ↑a + 1" },
-       type := { pp? := .some "Nat → Int" },
-     }:
-      Protocol.EnvInspectResult)
-  ]
+  let name3 := "Pantograph.mystery3"
+  step "env.add"
+    ({
+      name := name1,
+      value := "λ (a b: Prop) => Or a b",
+      isTheorem := false
+    }: Protocol.EnvAdd)
+   ({}: Protocol.EnvAddResult)
+  step "env.inspect" ({name := name1, value? := .some true} : Protocol.EnvInspect)
+   ({
+     value? := .some { pp? := .some "fun a b => a ∨ b" },
+     type := { pp? := .some "Prop → Prop → Prop" },
+   }: Protocol.EnvInspectResult)
+  step "env.add"
+    ({
+      name := name2,
+      type? := "Nat → Int",
+      value := "λ (a: Nat) => a + 1",
+      isTheorem := false
+    }: Protocol.EnvAdd)
+   ({}: Protocol.EnvAddResult)
+  step "env.inspect" ({name := name2, value? := .some true} : Protocol.EnvInspect)
+   ({
+     value? := .some { pp? := .some "fun a => ↑a + 1" },
+     type := { pp? := .some "Nat → Int" },
+   }: Protocol.EnvInspectResult)
+  step "env.add"
+    ({
+      name := name3,
+      levels? := .some #["u"]
+      type? := "(α : Type u) → α → (α × α)",
+      value := "λ (α : Type u) (x : α) => (x, x)",
+      isTheorem := false
+    }: Protocol.EnvAdd)
+   ({}: Protocol.EnvAddResult)
+  step "env.inspect" ({name := name3} : Protocol.EnvInspect)
+   ({
+     type := { pp? := .some "(α : Type u) → α → α × α" },
+   }: Protocol.EnvInspectResult)
 
 example : ∀ (p: Prop), p → p := by
   intro p h
   exact h
 
-def test_frontend_process : Test :=
-  [
-    let file := "example : ∀ (p q: Prop), p → p ∨ q := by\n  intro p q h\n  exact Or.inl h"
-    let goal1 := "p q : Prop\nh : p\n⊢ p ∨ q"
-    step "frontend.process"
-      [
-        ("file",              .str file),
-        ("invocations",       .bool true),
-        ("sorrys",            .bool false),
-        ("typeErrorsAsGoals", .bool false),
-        ("newConstants",      .bool false),
-      ]
-     ({
-       units := [{
-         boundary := (0, file.utf8ByteSize),
-         invocations? := .some [
-           {
-             goalBefore := "⊢ ∀ (p q : Prop), p → p ∨ q",
-             goalAfter := goal1,
-             tactic := "intro p q h",
-             usedConstants := #[],
-           },
-           {
-             goalBefore := goal1 ,
-             goalAfter := "",
-             tactic := "exact Or.inl h",
-             usedConstants := #["Or.inl"],
-           },
-         ]
-       }],
-    }: Protocol.FrontendProcessResult),
-  ]
+def test_frontend_process : Test := do
+  let file := "example : ∀ (p q: Prop), p → p ∨ q := by\n  intro p q h\n  exact Or.inl h"
+  let goal1 := "p q : Prop\nh : p\n⊢ p ∨ q"
+  IO.FS.withTempDir λ tempdir => do
+  let filename := s!"{tempdir}/invocations.jsonl"
+  step "frontend.process"
+    ({
+      file? := .some file,
+      invocations? := .some filename,
+    }: Protocol.FrontendProcess)
+   ({
+     units := [{
+       boundary := (0, file.utf8ByteSize),
+       nInvocations? := .some 2,
+     }],
+   }: Protocol.FrontendProcessResult)
+  stepFile (α := Protocol.FrontendData) "invocations" filename
+    { units := [{
+        invocations? := .some [
+          {
+            goalBefore := "⊢ ∀ (p q : Prop), p → p ∨ q",
+            goalAfter := goal1,
+            tactic := "intro p q h",
+            usedConstants := #[],
+          },
+          {
+            goalBefore := goal1 ,
+            goalAfter := "",
+            tactic := "exact Or.inl h",
+            usedConstants := #["Or.inl"],
+          },
+        ]
+    } ] }
 
 example : 1 + 2 = 3 := rfl
 example (p: Prop): p → p := by simp
 
-def test_frontend_process_sorry : Test :=
+def test_frontend_process_sorry : Test := do
   let solved := "example : 1 + 2 = 3 := rfl\n"
   let withSorry := "example (p: Prop): p → p := sorry"
-  [
-    let file := s!"{solved}{withSorry}"
-    let goal1: Protocol.Goal := {
-      name := "_uniq.6",
-      target := { pp? := .some "p → p" },
-      vars := #[{ name := "_uniq.4", userName := "p", type? := .some { pp? := .some "Prop" }}],
-    }
-    step "frontend.process"
-      [
-        ("file", .str file),
-        ("invocations", .bool false),
-        ("sorrys", .bool true),
-        ("typeErrorsAsGoals", .bool false),
-        ("newConstants", .bool false),
-      ]
-     ({
-       units := [{
-         boundary := (0, solved.utf8ByteSize),
-       }, {
-         boundary := (solved.utf8ByteSize, solved.utf8ByteSize + withSorry.utf8ByteSize),
-         goalStateId? := .some 0,
-         goals? := .some #[goal1],
-         goalSrcBoundaries? := .some #[(57, 62)],
-         messages := #["<anonymous>:2:0: warning: declaration uses 'sorry'\n"],
+  let file := s!"{solved}{withSorry}"
+  let goal1: Protocol.Goal := {
+    name := "_uniq.6",
+    target := { pp? := .some "p → p" },
+    vars := #[{ name := "_uniq.4", userName := "p", type? := .some { pp? := .some "Prop" }}],
+  }
+  step "frontend.process"
+    ({
+      file? := .some file,
+      sorrys := true,
+    }: Protocol.FrontendProcess)
+   ({
+     units := [{
+       boundary := (0, solved.utf8ByteSize),
+     }, {
+       boundary := (solved.utf8ByteSize, solved.utf8ByteSize + withSorry.utf8ByteSize),
+       goalStateId? := .some 0,
+       goals? := .some #[goal1],
+       goalSrcBoundaries? := .some #[(57, 62)],
+       messages := #[{
+         fileName := "<anonymous>",
+         kind := `hasSorry,
+         pos := ⟨2, 0⟩,
+         endPos := .some ⟨2, 7⟩,
+         severity := .warning,
+         data := "declaration uses 'sorry'",
        }],
-    }: Protocol.FrontendProcessResult),
-  ]
+     }],
+   }: Protocol.FrontendProcessResult)
 
+def test_import_open : Test := do
+  let header := "import Init\nopen Nat\nuniverse u"
+  let goal1: Protocol.Goal := {
+    name := "_uniq.81",
+    target := { pp? := .some "n + 1 = n.succ" },
+    vars := #[{ name := "_uniq.80", userName := "n", type? := .some { pp? := .some "Nat" }}],
+  }
+  step "frontend.process"
+    ({
+      file? := .some header,
+      readHeader := true,
+      inheritEnv := true,
+    }: Protocol.FrontendProcess)
+   ({
+     units := [
+       { boundary := (12, 21) },
+       { boundary := (21, header.utf8ByteSize) },
+     ],
+   }: Protocol.FrontendProcessResult)
+  step "goal.start" ({ expr := "∀ (n : Nat), n + 1 = Nat.succ n"} : Protocol.GoalStart)
+   ({ stateId := 0, root := "_uniq.79" }: Protocol.GoalStartResult)
+  step "goal.tactic" ({ stateId := 0, tactic? := .some "intro n" }: Protocol.GoalTactic)
+   ({ nextStateId? := .some 1, goals? := #[goal1], }: Protocol.GoalTacticResult)
+  step "goal.tactic" ({ stateId := 1, tactic? := .some "apply add_one" }: Protocol.GoalTactic)
+   ({ nextStateId? := .some 2, goals? := .some #[], }: Protocol.GoalTacticResult)
+  step "goal.start" ({ expr := "∀ (x : Sort u), Sort (u + 1)"} : Protocol.GoalStart)
+   ({ stateId := 3, root := "_uniq.5" }: Protocol.GoalStartResult)
 
-def runTest (env: Lean.Environment) (steps: Test): IO LSpec.TestSeq := do
+/-- Ensure there cannot be circular references -/
+def test_frontend_process_circular : Test := do
+  let withSorry := "theorem mystery : 1 + 2 = 2 + 3 := sorry"
+  let goal1: Protocol.Goal := {
+    name := "_uniq.2",
+    target := { pp? := .some "1 + 2 = 2 + 3" },
+    vars := #[],
+  }
+  step "frontend.process"
+    ({
+      file? := .some withSorry,
+      sorrys := true,
+    }: Protocol.FrontendProcess)
+   ({
+     units := [{
+       boundary := (0, withSorry.utf8ByteSize),
+       goalStateId? := .some 0,
+       goals? := .some #[goal1],
+       goalSrcBoundaries? := .some #[(35, 40)],
+       messages := #[{
+         fileName := "<anonymous>",
+         kind := `hasSorry,
+         pos := ⟨1, 8⟩,
+         endPos := .some ⟨1, 15⟩,
+         severity := .warning,
+         data := "declaration uses 'sorry'"
+       }],
+     }],
+   } : Protocol.FrontendProcessResult)
+  step "goal.tactic" ({ stateId := 0, tactic? := .some "exact?" }: Protocol.GoalTactic)
+   ({
+      messages? := .some #[{
+        fileName := ← getFileName,
+        kind := .anonymous,
+        pos := ⟨0, 0⟩,
+        data := "`exact?` could not close the goal. Try `apply?` to see partial suggestions."
+      }]
+    } : Protocol.GoalTacticResult)
+
+def runTestSuite (env : Lean.Environment) (steps : Test): IO LSpec.TestSeq := do
   -- Setup the environment for execution
-  let context: Context := {}
-  let commands: MainM LSpec.TestSeq :=
-    steps.foldlM (λ suite step => do
-      let result ← step
-      return suite ++ result) LSpec.TestSeq.done
-  runCoreMSeq env <| commands.run context |>.run' {}
-
+  let coreContext ← createCoreContext #[]
+  let mainM : MainM LSpec.TestSeq := runTest steps
+  mainM.run { coreContext } |>.run' { env }
 
 def suite (env : Lean.Environment): List (String × IO LSpec.TestSeq) :=
   let tests := [
-    ("expr.echo", test_elab),
+    ("expr.echo", test_expr_echo),
     ("options.set options.print", test_option_modify),
     ("Malformed command", test_malformed_command),
     ("Tactic", test_tactic),
+    ("Tactic Timeout", test_tactic_timeout),
     ("Manual Mode", test_automatic_mode false),
     ("Automatic Mode", test_automatic_mode true),
+    ("Conv-Calc", test_conv_calc),
     ("env.add env.inspect", test_env_add_inspect),
     ("frontend.process invocation", test_frontend_process),
     ("frontend.process sorry", test_frontend_process_sorry),
+    ("frontend.process import", test_import_open),
+    ("frontend.process circular", test_frontend_process_circular),
   ]
-  tests.map (fun (name, test) => (name, runTest env test))
+  tests.map (fun (name, test) => (name, runTestSuite env test))
 
 
 end Pantograph.Test.Integration

Test/Library.lean

@@ -8,15 +8,18 @@ open Pantograph
 
 namespace Pantograph.Test.Library
 
+def runTermElabM { α } (termElabM: Elab.TermElabM α): CoreM α :=
+  termElabM.run' (ctx := defaultElabContext) |>.run'
+
 def test_expr_echo (env: Environment): IO LSpec.TestSeq := do
   let inner: CoreM LSpec.TestSeq := do
     let prop_and_proof := "⟨∀ (x: Prop), x → x, λ (x: Prop) (h: x) => h⟩"
     let tests := LSpec.TestSeq.done
-    let echoResult ← exprEcho prop_and_proof (options := {})
+    let echoResult ← runTermElabM $ exprEcho prop_and_proof (options := {})
     let tests := tests.append (LSpec.test "fail" (echoResult.toOption == .some {
       type := { pp? := "?m.2" }, expr := { pp? := "?m.3" }
     }))
-    let echoResult ← exprEcho prop_and_proof (expectedType? := .some "Σ' p:Prop, p") (options := { printExprAST := true })
+    let echoResult ← runTermElabM $ exprEcho prop_and_proof (expectedType? := .some "Σ' p:Prop, p") (options := { printExprAST := true })
     let tests := tests.append (LSpec.test "fail" (echoResult.toOption == .some {
       type := {
         pp? := "(p : Prop) ×' p",

Test/Main.lean

@@ -1,4 +1,3 @@
-import LSpec
 import Test.Delate
 import Test.Environment
 import Test.Frontend
@@ -9,6 +8,8 @@ import Test.Proofs
 import Test.Serial
 import Test.Tactic
 
+import LSpec
+
 -- Test running infrastructure
 
 namespace Pantograph.Test
@@ -17,9 +18,9 @@ def addPrefix (pref: String) (tests: List (String × α)): List (String  × α)
   tests.map (λ (name, x) => (pref ++ "/" ++ name, x))
 
 /-- Runs test in parallel. Filters test name if given -/
-def runTestGroup (filter: Option String) (tests: List (String × IO LSpec.TestSeq)): IO LSpec.TestSeq := do
-  let tests: List (String × IO LSpec.TestSeq) := match filter with
-    | .some filter => tests.filter (λ (name, _) => filter.isPrefixOf name)
+def runTestGroup (nameFilter?: Option String) (tests: List (String × IO LSpec.TestSeq)): IO LSpec.TestSeq := do
+  let tests: List (String × IO LSpec.TestSeq) := match nameFilter? with
+    | .some nameFilter => tests.filter (λ (name, _) => nameFilter.isPrefixOf name)
     | .none => tests
   let tasks: List (String × Task _) ← tests.mapM (λ (name, task) => do
     return (name, ← EIO.asTask task))
@@ -37,27 +38,30 @@ open Pantograph.Test
 
 /-- Main entry of tests; Provide an argument to filter tests by prefix -/
 def main (args: List String) := do
-  let name_filter := args.head?
+  let nameFilter? := args.head?
   Lean.initSearchPath (← Lean.findSysroot)
-  let env_default: Lean.Environment ← Lean.importModules
+  let env_default : Lean.Environment ← Lean.importModules
     (imports := #[`Init])
     (opts := {})
     (trustLevel := 1)
+    (loadExts := true)
 
-  let suites: List (String × List (String × IO LSpec.TestSeq)) := [
+  let suites: List (String × (Lean.Environment → List (String × IO LSpec.TestSeq))) := [
     ("Environment", Environment.suite),
-    ("Frontend", Frontend.suite env_default),
-    ("Integration", Integration.suite env_default),
-    ("Library", Library.suite env_default),
-    ("Metavar", Metavar.suite env_default),
-    ("Proofs", Proofs.suite env_default),
-    ("Delate", Delate.suite env_default),
-    ("Serial", Serial.suite env_default),
-    ("Tactic/Assign", Tactic.Assign.suite env_default),
-    ("Tactic/Congruence", Tactic.Congruence.suite env_default),
-    ("Tactic/Motivated Apply", Tactic.MotivatedApply.suite env_default),
-    ("Tactic/No Confuse", Tactic.NoConfuse.suite env_default),
-    ("Tactic/Prograde", Tactic.Prograde.suite env_default),
+    ("Frontend", Frontend.suite),
+    ("Integration", Integration.suite),
+    ("Library", Library.suite),
+    ("Metavar", Metavar.suite),
+    ("Proofs", Proofs.suite),
+    ("Delate", Delate.suite),
+    ("Serial", Serial.suite),
+    ("Tactic/Assign", Tactic.Assign.suite),
+    ("Tactic/Fragment", Tactic.Fragment.suite),
+    ("Tactic/Prograde", Tactic.Prograde.suite),
+    ("Tactic/Special", Tactic.Special.suite),
   ]
-  let tests: List (String × IO LSpec.TestSeq) := suites.foldl (λ acc (name, suite) => acc ++ (addPrefix name suite)) []
-  LSpec.lspecIO (← runTestGroup name_filter tests)
+  let suiterunner (f : Lean.Environment → List (String × IO LSpec.TestSeq)) :=
+    f env_default
+  let tests : List (String × IO LSpec.TestSeq) := suites.foldl (init := []) λ acc (name, suite) =>
+    acc ++ (addPrefix name $ suiterunner suite)
+  LSpec.lspecEachIO [()] (λ () => runTestGroup nameFilter? tests)

Test/Metavar.lean

@@ -25,8 +25,8 @@ def test_instantiate_mvar: TestM Unit := do
       addTest $ assertUnreachable e
       return ()
   let t ← Lean.Meta.inferType expr
-  addTest $ LSpec.check "typing" ((toString (← serializeExpressionSexp t)) =
-    "((:c LE.le) (:c Nat) (:c instLENat) ((:c OfNat.ofNat) (:mv _uniq.2) (:lit 2) (:mv _uniq.3)) ((:c OfNat.ofNat) (:mv _uniq.14) (:lit 5) (:mv _uniq.15)))")
+  checkEq "typing" (toString (← serializeExpressionSexp t))
+    "((:c LE.le) (:c Nat) (:c instLENat) ((:c OfNat.ofNat) (:mv _uniq.2) (:lit 2) (:mv _uniq.3)) ((:c OfNat.ofNat) (:mv _uniq.15) (:lit 5) (:mv _uniq.16)))"
   return ()
 
 def startProof (expr: String): TestM (Option GoalState) := do
@@ -79,20 +79,20 @@ def test_m_couple: TestM Unit := do
       return ()
 
   let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := "apply Nat.le_trans") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   addTest $ LSpec.check "apply Nat.le_trans" ((← state1.serializeGoals (options := ← read)).map (·.target.pp?) =
     #[.some "2 ≤ ?m", .some "?m ≤ 5", .some "Nat"])
-  addTest $ LSpec.test "(1 root)" state1.rootExpr?.isNone
+  checkTrue "(1 root)" $ ¬ state1.isSolved
   -- Set m to 3
   let state2 ← match ← state1.tacticOn (goalId := 2) (tactic := "exact 3") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
-  addTest $ LSpec.test "(1b root)" state2.rootExpr?.isNone
+  checkTrue "(1b root)" $ ¬ state2.isSolved
   let state1b ← match state2.continue state1 with
     | .error msg => do
       addTest $ assertUnreachable $ msg
@@ -100,7 +100,7 @@ def test_m_couple: TestM Unit := do
     | .ok state => pure state
   addTest $ LSpec.check "exact 3" ((← state1b.serializeGoals (options := ← read)).map (·.target.pp?) =
     #[.some "2 ≤ 3", .some "3 ≤ 5"])
-  addTest $ LSpec.test "(2 root)" state1b.rootExpr?.isNone
+  checkTrue "(2 root)" $ ¬ state1b.isSolved
 
 def test_m_couple_simp: TestM Unit := do
   let state? ← startProof "(2: Nat) ≤ 5"
@@ -111,22 +111,23 @@ def test_m_couple_simp: TestM Unit := do
       return ()
 
   let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := "apply Nat.le_trans") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   let serializedState1 ← state1.serializeGoals (options := { ← read with printDependentMVars := true })
   addTest $ LSpec.check "apply Nat.le_trans" (serializedState1.map (·.target.pp?) =
     #[.some "2 ≤ ?m", .some "?m ≤ 5", .some "Nat"])
+  let n := state1.goals[2]!
   addTest $ LSpec.check "(metavariables)" (serializedState1.map (·.target.dependentMVars?.get!) =
-    #[#["_uniq.38"], #["_uniq.38"], #[]])
+    #[#[toString n.name], #[toString n.name], #[]])
 
   let state2 ← match ← state1.tacticOn (goalId := 2) (tactic := "exact 2") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
-  addTest $ LSpec.test "(1b root)" state2.rootExpr?.isNone
+  checkTrue "(1b root)" $ ¬ state2.isSolved
   let state1b ← match state2.continue state1 with
     | .error msg => do
       addTest $ assertUnreachable $ msg
@@ -134,9 +135,9 @@ def test_m_couple_simp: TestM Unit := do
     | .ok state => pure state
   addTest $ LSpec.check "exact 2" ((← state1b.serializeGoals (options := ← read)).map (·.target.pp?) =
     #[.some "2 ≤ 2", .some "2 ≤ 5"])
-  addTest $ LSpec.test "(2 root)" state1b.rootExpr?.isNone
+  checkTrue "(2 root)" $ ¬ state1b.isSolved
   let state3 ← match ← state1b.tacticOn (goalId := 0) (tactic := "simp") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -146,7 +147,7 @@ def test_m_couple_simp: TestM Unit := do
       return ()
     | .ok state => pure state
   let state5 ← match ← state4.tacticOn (goalId := 0) (tactic := "simp") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -158,10 +159,10 @@ def test_m_couple_simp: TestM Unit := do
       addTest $ assertUnreachable "(5 root)"
       return ()
   let rootStr: String := toString (← Lean.Meta.ppExpr root)
-  addTest $ LSpec.check "(5 root)" (rootStr = "Nat.le_trans (of_eq_true (Init.Data.Nat.Basic._auxLemma.4 2)) (of_eq_true (eq_true_of_decide (Eq.refl true)))")
+  --checkEq "(5 root)" rootStr "Nat.le_trans (of_eq_true (_proof_4✝ 2)) (of_eq_true (eq_true_of_decide (Eq.refl true)))"
   let unfoldedRoot ←  unfoldAuxLemmas root
-  addTest $ LSpec.check "(5 root)" ((toString (← Lean.Meta.ppExpr unfoldedRoot)) =
-    "Nat.le_trans (of_eq_true (eq_true (Nat.le_refl 2))) (of_eq_true (eq_true_of_decide (Eq.refl true)))")
+  checkEq "(5 root)" (toString (← Lean.Meta.ppExpr unfoldedRoot))
+    "Nat.le_trans (of_eq_true (eq_true (Nat.le_refl 2))) (of_eq_true (eq_true_of_decide (Eq.refl true)))"
   return ()
 
 def test_proposition_generation: TestM Unit := do
@@ -173,7 +174,7 @@ def test_proposition_generation: TestM Unit := do
       return ()
 
   let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := "apply PSigma.mk") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -184,27 +185,27 @@ def test_proposition_generation: TestM Unit := do
       ])
   if let #[goal1, goal2] := ← state1.serializeGoals (options := { (← read) with printExprAST := true }) then
     addTest $ LSpec.test "(1 reference)" (goal1.target.sexp? = .some s!"(:mv {goal2.name})")
-  addTest $ LSpec.test "(1 root)" state1.rootExpr?.isNone
+  checkTrue "(1 root)" $ ¬ state1.isSolved
 
   let state2 ← match ← state1.tryAssign (state1.get! 0) (expr := "λ (x: Nat) => _") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   addTest $ LSpec.check ":= λ (x: Nat), _" ((← state2.serializeGoals (options := ← read)).map (·.target.pp?) =
-    #[.some "?m.29 x"])
-  addTest $ LSpec.test "(2 root)" state2.rootExpr?.isNone
+    #[.some "?m.30 x"])
+  checkTrue "(2 root)" $ ¬ state2.isSolved
 
   let assign := "Eq.refl x"
   let state3 ← match ← state2.tryAssign (state2.get! 0) (expr := assign) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   addTest $ LSpec.check s!":= {assign}" ((← state3.serializeGoals (options := ← read)).map (·.target.pp?) =
     #[])
 
-  addTest $ LSpec.test "(3 root)" state3.rootExpr?.isSome
+  checkTrue "(3 root)" state3.isSolved
   return ()
 
 def test_partial_continuation: TestM Unit := do
@@ -216,7 +217,7 @@ def test_partial_continuation: TestM Unit := do
       return ()
 
   let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := "apply Nat.le_trans") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -224,7 +225,7 @@ def test_partial_continuation: TestM Unit := do
     #[.some "2 ≤ ?m", .some "?m ≤ 5", .some "Nat"])
 
   let state2 ← match ← state1.tacticOn (goalId := 2) (tactic := "apply Nat.succ") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -238,34 +239,74 @@ def test_partial_continuation: TestM Unit := do
       addTest $ assertUnreachable $ msg
       return ()
     | .ok state => pure state
-  addTest $ LSpec.check "(continue)" ((← state1b.serializeGoals (options := ← read)).map (·.target.pp?) =
+  addTest $ LSpec.check "(continue 1)" ((← state1b.serializeGoals (options := ← read)).map (·.target.pp?) =
     #[.some "2 ≤ Nat.succ ?m", .some "Nat.succ ?m ≤ 5", .some "Nat"])
-  addTest $ LSpec.test "(2 root)" state1b.rootExpr?.isNone
+  checkTrue "(2 root)" state1b.rootExpr?.get!.hasExprMVar
 
   -- Roundtrip
   --let coupled_goals := coupled_goals.map (λ g =>
   --  { name := str_to_name $ serializeName g.name (sanitize := false)})
-  let coupled_goals := coupled_goals.map (λ g => serializeName g.name (sanitize := false))
-  let coupled_goals := coupled_goals.map (λ g => { name := g.toName })
+  let coupled_goals := coupled_goals.map (·.name.toString)
+  let coupled_goals := coupled_goals.map ({ name := ·.toName })
   let state1b ← match state2.resume (goals := coupled_goals) with
     | .error msg => do
       addTest $ assertUnreachable $ msg
       return ()
     | .ok state => pure state
-  addTest $ LSpec.check "(continue)" ((← state1b.serializeGoals (options := ← read)).map (·.target.pp?) =
-    #[.some "2 ≤ Nat.succ ?m", .some "Nat.succ ?m ≤ 5", .some "Nat"])
-  addTest $ LSpec.test "(2 root)" state1b.rootExpr?.isNone
+  checkEq "(continue 2)" ((← state1b.serializeGoals (options := ← read)).map (·.target.pp?))
+    #[.some "2 ≤ Nat.succ ?m", .some "Nat.succ ?m ≤ 5", .some "Nat"]
+  checkTrue "(2 root)" state1b.rootExpr?.get!.hasExprMVar
 
   -- Continuation should fail if the state does not exist:
   match state0.resume coupled_goals with
-  | .error error => addTest $ LSpec.check "(continuation failure message)" (error = "Goals [_uniq.40, _uniq.41, _uniq.38, _uniq.47] are not in scope")
-  | .ok _ => addTest $ assertUnreachable "(continuation failure)"
+  | .error error => checkEq "(continuation failure message)" error "Goals [_uniq.45, _uniq.46, _uniq.43, _uniq.52] are not in scope"
+  | .ok _ => fail "(continuation should fail)"
   -- Continuation should fail if some goals have not been solved
   match state2.continue state1 with
-  | .error error => addTest $ LSpec.check "(continuation failure message)" (error = "Target state has unresolved goals")
-  | .ok _ => addTest $ assertUnreachable "(continuation failure)"
+  | .error error => checkEq "(continuation failure message)" error "Target state has unresolved goals"
+  | .ok _ => fail "(continuation should fail)"
   return ()
 
+def test_branch_unification : TestM Unit := do
+  let .ok rootTarget ← elabTerm (← `(term|∀ (p q : Prop), p → p ∧ (p ∨ q))) .none | unreachable!
+  let state ← GoalState.create rootTarget
+  let .success state _  ← state.tacticOn' 0 (← `(tactic|intro p q h)) | fail "intro failed to run"
+  let .success state _  ← state.tacticOn' 0 (← `(tactic|apply And.intro)) | fail "apply And.intro failed to run"
+  let .success state1 _  ← state.tacticOn' 0 (← `(tactic|exact h)) | fail "exact h failed to run"
+  let .success state2 _  ← state.tacticOn' 1 (← `(tactic|apply Or.inl)) | fail "apply Or.inl failed to run"
+  checkEq "(state2 goals)" state2.goals.length 1
+  let state' ← state2.replay state state1
+  checkEq "(state' goals)" state'.goals.length 1
+  let .success stateT _ ← state'.tacticOn' 0 (← `(tactic|exact h)) | fail "exact h failed to run"
+  let .some root := stateT.rootExpr? | fail "Root expression must exist"
+  checkEq "(root)" (toString $ ← Meta.ppExpr root) "fun p q h => ⟨h, Or.inl h⟩"
+
+/-- Test merger when both branches have new aux lemmas -/
+def test_replay_environment : TestM Unit := do
+  let .ok rootTarget ← elabTerm (← `(term|(2: Nat) ≤ 3 ∧ (3: Nat) ≤ 5)) .none | unreachable!
+  let state ← GoalState.create rootTarget
+  let .success state _  ← state.tacticOn' 0 (← `(tactic|apply And.intro)) | fail "apply And.intro failed to run"
+  let goal := state.goals[0]!
+  let type ← goal.withContext do
+    let .ok type ← elabTerm (← `(term|(2: Nat) ≤ 3)) (.some $ .sort 0) | unreachable!
+    pure type
+  let .success state1 _  ← state.tryTacticM goal (Tactic.assignWithAuxLemma type) | fail "left"
+
+  state.restoreMetaM
+  let goal := state.goals[1]!
+  let type ← goal.withContext do
+    let .ok type ← elabTerm (← `(term|(3: Nat) ≤ 5)) (.some $ .sort 0) | unreachable!
+    pure type
+  let .success state2 _  ← state.tryTacticM goal (Tactic.assignWithAuxLemma type) | fail "right"
+  checkEq "(state1 goals)" state1.goals.length 0
+  checkEq "(state2 goals)" state2.goals.length 0
+  let stateT ← state2.replay state state1
+  checkEq "(stateT goals)" stateT.goals.length 0
+  let .some root := stateT.rootExpr? | fail "Root expression must exist"
+  checkTrue "root has aux lemma" $ root.getUsedConstants.any isAuxLemma
+  checkEq "(root)" (toString $ ← Meta.ppExpr root) "⟨_proof_1, _proof_2⟩"
+  let root ← unfoldAuxLemmas root
+  checkEq "(root unfold)" (toString $ ← Meta.ppExpr root) "⟨sorry, sorry⟩"
 
 def suite (env: Environment): List (String × IO LSpec.TestSeq) :=
   let tests := [
@@ -273,7 +314,9 @@ def suite (env: Environment): List (String × IO LSpec.TestSeq) :=
     ("2 < 5", test_m_couple),
     ("2 < 5", test_m_couple_simp),
     ("Proposition Generation", test_proposition_generation),
-    ("Partial Continuation", test_partial_continuation)
+    ("Partial Continuation", test_partial_continuation),
+    ("Branch Unification", test_branch_unification),
+    --("Replay Environment", test_replay_environment),
   ]
   tests.map (fun (name, test) => (name, proofRunner env test))
 

Test/Proofs.lean

@@ -11,7 +11,7 @@ open Pantograph
 open Lean
 
 inductive Start where
-  | copy (name: String) -- Start from some name in the environment
+  | copy (name: Name) -- Start from some name in the environment
   | expr (expr: String) -- Start from some expression
 
 abbrev TestM := TestT $ ReaderT Protocol.Options $ Elab.TermElabM
@@ -20,7 +20,7 @@ def startProof (start: Start): TestM (Option GoalState) := do
   let env ← Lean.MonadEnv.getEnv
   match start with
   | .copy name =>
-    let cInfo? := name.toName |> env.find?
+    let cInfo? := name |> env.find?
     addTest $ LSpec.check s!"Symbol exists {name}" cInfo?.isSome
     match cInfo? with
     | .some cInfo =>
@@ -29,24 +29,10 @@ def startProof (start: Start): TestM (Option GoalState) := do
     | .none =>
       return Option.none
   | .expr expr =>
-    let syn? := parseTerm env expr
-    addTest $ LSpec.check s!"Parsing {expr}" (syn?.isOk)
-    match syn? with
-    | .error error =>
-      IO.println error
-      return Option.none
-    | .ok syn =>
-      let expr? ← elabType syn
-      addTest $ LSpec.check s!"Elaborating" expr?.isOk
-      match expr? with
-      | .error error =>
-        IO.println error
-        return Option.none
-      | .ok expr =>
-        let goal ← GoalState.create (expr := expr)
-        return Option.some goal
+    let expr ← parseSentence expr
+    return .some $ ← GoalState.create (expr := expr)
 
-def buildNamedGoal (name: String) (nameType: List (String × String)) (target: String)
+private def buildNamedGoal (name: String) (nameType: List (String × String)) (target: String)
     (userName?: Option String := .none): Protocol.Goal :=
   {
     name,
@@ -57,7 +43,7 @@ def buildNamedGoal (name: String) (nameType: List (String × String)) (target: S
       type? := .some { pp? := .some x.snd },
     })).toArray
   }
-def buildGoal (nameType: List (String × String)) (target: String) (userName?: Option String := .none):
+private def buildGoal (nameType: List (String × String)) (target: String) (userName?: Option String := .none):
     Protocol.Goal :=
   {
     userName?,
@@ -67,7 +53,7 @@ def buildGoal (nameType: List (String × String)) (target: String) (userName?: O
       type? := .some { pp? := .some x.snd },
     })).toArray
   }
-def proofRunner (env: Lean.Environment) (tests: TestM Unit): IO LSpec.TestSeq := do
+private def proofRunner (env: Lean.Environment) (tests: TestM Unit): IO LSpec.TestSeq := do
   let termElabM := tests.run LSpec.TestSeq.done |>.run {} -- with default options
 
   let coreContext: Lean.Core.Context ← createCoreContext #[]
@@ -80,24 +66,18 @@ def proofRunner (env: Lean.Environment) (tests: TestM Unit): IO LSpec.TestSeq :=
     return a
 
 def test_identity: TestM Unit := do
-  let state? ← startProof (.expr "∀ (p: Prop), p → p")
-  let state0 ← match state? with
-    | .some state => pure state
-    | .none => do
-      addTest $ assertUnreachable "Goal could not parse"
-      return ()
-
-  let tactic := "intro p h"
-  let state1 ← match ← state0.tacticOn 0 tactic with
-    | .success state => pure state
+  let rootTarget ← Elab.Term.elabTerm (← `(term|∀ (p: Prop), p → p)) .none
+  let state0 ← GoalState.create (expr := rootTarget)
+  let state1 ← match ← state0.tacticOn' 0 (← `(tactic|intro p h)) with
+    | .success state _ => pure state
     | other => do
-      addTest $ assertUnreachable $ other.toString
+      fail other.toString
       return ()
-  let inner := "_uniq.12"
-  addTest $ LSpec.check tactic ((← state1.serializeGoals (options := ← read)).map (·.name) =
+  let inner := "_uniq.11"
+  addTest $ LSpec.check "intro" ((← state1.serializeGoals (options := ← read)).map (·.name) =
     #[inner])
   let state1parent ← state1.withParentContext do
-    serializeExpressionSexp (← instantiateAll state1.parentExpr?.get!)
+    serializeExpressionSexp (← instantiateAll state1.parentExpr!)
   addTest $ LSpec.test "(1 parent)" (state1parent == s!"(:lambda p (:sort 0) (:lambda h 0 (:subst (:mv {inner}) 1 0)))")
 
 -- Individual test cases
@@ -106,7 +86,7 @@ example: ∀ (a b: Nat), a + b = b + a := by
   rw [Nat.add_comm]
 def test_nat_add_comm (manual: Bool): TestM Unit := do
   let state? ← startProof <| match manual with
-    | false => .copy "Nat.add_comm"
+    | false => .copy `Nat.add_comm
     | true => .expr "∀ (a b: Nat), a + b = b + a"
   addTest $ LSpec.check "Start goal" state?.isSome
   let state0 ← match state? with
@@ -116,7 +96,7 @@ def test_nat_add_comm (manual: Bool): TestM Unit := do
       return ()
 
   let state1 ← match ← state0.tacticOn 0 "intro n m" with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -125,53 +105,20 @@ def test_nat_add_comm (manual: Bool): TestM Unit := do
 
   match ← state1.tacticOn 0 "assumption" with
   | .failure #[message] =>
-    addTest $ LSpec.check "assumption" (message = "tactic 'assumption' failed\nn m : Nat\n⊢ n + m = m + n")
+    checkEq "assumption"
+      (← message.toString)
+      s!"{← getFileName}:0:0: error: tactic 'assumption' failed\nn m : Nat\n⊢ n + m = m + n\n"
   | other => do
-    addTest $ assertUnreachable $ other.toString
+    addTest $ assertUnreachable other.toString
 
   let state2 ← match ← state1.tacticOn 0 "rw [Nat.add_comm]" with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   addTest $ LSpec.test "rw [Nat.add_comm]" state2.goals.isEmpty
 
   return ()
-def test_delta_variable: TestM Unit := do
-  let options: Protocol.Options := { noRepeat := true }
-  let state? ← startProof <| .expr "∀ (a b: Nat), a + b = b + a"
-  addTest $ LSpec.check "Start goal" state?.isSome
-  let state0 ← match state? with
-    | .some state => pure state
-    | .none => do
-      addTest $ assertUnreachable "Goal could not parse"
-      return ()
-
-  let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := "intro n") with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check "intro n" ((← state1.serializeGoals (parent := state0) options).map (·.devolatilize) =
-    #[buildGoalSelective [("n", .some "Nat")] "∀ (b : Nat), n + b = b + n"])
-  let state2 ← match ← state1.tacticOn (goalId := 0) (tactic := "intro m") with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check "intro m" ((← state2.serializeGoals (parent := state1) options).map (·.devolatilize) =
-    #[buildGoalSelective [("n", .none), ("m", .some "Nat")] "n + m = m + n"])
-  return ()
-  where
--- Like `buildGoal` but allow certain variables to be elided.
-  buildGoalSelective (nameType: List (String × Option String)) (target: String): Protocol.Goal :=
-    {
-      target := { pp? := .some target},
-      vars := (nameType.map fun x => ({
-        userName := x.fst,
-        type? := x.snd.map (λ type => { pp? := type }),
-      })).toArray
-    }
 
 example (w x y z : Nat) (p : Nat → Prop)
         (h : p (x * y + z * w * x)) : p (x * w * z + y * x) := by
@@ -187,27 +134,27 @@ def test_arith: TestM Unit := do
 
   let tactic := "intros"
   let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   addTest $ LSpec.check tactic (state1.goals.length = 1)
-  addTest $ LSpec.test "(1 root)" state1.rootExpr?.isNone
+  checkTrue "(1 root)" state1.rootExpr?.get!.hasExprMVar
   let state2 ← match ← state1.tacticOn (goalId := 0) (tactic := "simp [Nat.add_assoc, Nat.add_comm, Nat.add_left_comm, Nat.mul_comm, Nat.mul_assoc, Nat.mul_left_comm] at *") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   addTest $ LSpec.check "simp ..." (state2.goals.length = 1)
-  addTest $ LSpec.check "(2 root)" state2.rootExpr?.isNone
+  checkTrue "(2 root)" state2.rootExpr?.get!.hasExprMVar
   let tactic := "assumption"
   let state3 ← match ← state2.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   addTest $ LSpec.test tactic state3.goals.isEmpty
-  addTest $ LSpec.check "(3 root)" state3.rootExpr?.isSome
+  checkTrue "(3 root)" $ ¬ state3.rootExpr?.get!.hasExprMVar
   return ()
 
 -- Two ways to write the same theorem
@@ -232,12 +179,12 @@ def test_or_comm: TestM Unit := do
     | .none => do
       addTest $ assertUnreachable "Goal could not parse"
       return ()
-  addTest $ LSpec.check "(0 parent)" state0.parentExpr?.isNone
-  addTest $ LSpec.check "(0 root)" state0.rootExpr?.isNone
+  checkTrue "(0 parent)" state0.parentMVars.isEmpty
+  checkTrue "(0 root)" state0.rootExpr?.isNone
 
   let tactic := "intro p q h"
   let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -257,15 +204,15 @@ def test_or_comm: TestM Unit := do
         { name := fvH, userName := "h", type? := .some { pp? := .some "p ∨ q" } }
       ]
     }])
-  addTest $ LSpec.check "(1 parent)" state1.parentExpr?.isSome
-  addTest $ LSpec.check "(1 root)" state1.rootExpr?.isNone
+  checkTrue "(1 parent)" state1.hasUniqueParent
+  checkTrue "(1 root)" $ ¬ state1.isSolved
 
   let state1parent ← state1.withParentContext do
-    serializeExpressionSexp (← instantiateAll state1.parentExpr?.get!)
+    serializeExpressionSexp (← instantiateAll state1.parentExpr!)
   addTest $ LSpec.test "(1 parent)" (state1parent == s!"(:lambda p (:sort 0) (:lambda q (:sort 0) (:lambda h ((:c Or) 1 0) (:subst (:mv {state1g0}) 2 1 0))))")
   let tactic := "cases h"
   let state2 ← match ← state1.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -276,11 +223,11 @@ def test_or_comm: TestM Unit := do
   let (caseL, caseR) := (state2g0.name.toString, state2g1.name.toString)
   addTest $ LSpec.check tactic ((← state2.serializeGoals (options := ← read)).map (·.name) =
     #[caseL, caseR])
-  addTest $ LSpec.check "(2 parent exists)" state2.parentExpr?.isSome
-  addTest $ LSpec.check "(2 root)" state2.rootExpr?.isNone
+  checkTrue "(2 parent exists)" state2.hasUniqueParent
+  checkTrue "(2 root)" $ ¬ state2.isSolved
 
   let state2parent ← state2.withParentContext do
-    serializeExpressionSexp (← instantiateAll state2.parentExpr?.get!)
+    serializeExpressionSexp (← instantiateAll state2.parentExpr!)
   let orPQ := s!"((:c Or) (:fv {fvP}) (:fv {fvQ}))"
   let orQP := s!"((:c Or) (:fv {fvQ}) (:fv {fvP}))"
   let motive := s!"(:lambda t {orPQ} (:forall h ((:c Eq) ((:c Or) (:fv {fvP}) (:fv {fvQ})) (:fv {fvH}) 0) {orQP}))"
@@ -291,57 +238,57 @@ def test_or_comm: TestM Unit := do
     s!"((:c Or.casesOn) (:fv {fvP}) (:fv {fvQ}) {motive} (:fv {fvH}) {caseL} {caseR} {conduit})")
 
   let state3_1 ← match ← state2.tacticOn (goalId := 0) (tactic := "apply Or.inr") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   let state3_1parent ← state3_1.withParentContext do
-    serializeExpressionSexp (← instantiateAll state3_1.parentExpr?.get!)
+    serializeExpressionSexp (← instantiateAll state3_1.parentExpr!)
   let [state3_1goal0] := state3_1.goals | fail "Should have 1 goal"
   addTest $ LSpec.test "(3_1 parent)" (state3_1parent == s!"((:c Or.inr) (:fv {fvQ}) (:fv {fvP}) (:mv {state3_1goal0}))")
   addTest $ LSpec.check "· apply Or.inr" (state3_1.goals.length = 1)
   let state4_1 ← match ← state3_1.tacticOn (goalId := 0) (tactic := "assumption") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   addTest $ LSpec.check "  assumption" state4_1.goals.isEmpty
-  let state4_1parent ← instantiateAll state4_1.parentExpr?.get!
+  let state4_1parent ← instantiateAll state4_1.parentExpr!
   addTest $ LSpec.test "(4_1 parent)" state4_1parent.isFVar
-  addTest $ LSpec.check "(4_1 root)" state4_1.rootExpr?.isNone
+  checkTrue "(4_1 root)" $ ¬ state4_1.isSolved
   let state3_2 ← match ← state2.tacticOn (goalId := 1) (tactic := "apply Or.inl") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   addTest $ LSpec.check "· apply Or.inl" (state3_2.goals.length = 1)
   let state4_2 ← match ← state3_2.tacticOn (goalId := 0) (tactic := "assumption") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   addTest $ LSpec.check "  assumption" state4_2.goals.isEmpty
-  addTest $ LSpec.check "(4_2 root)" state4_2.rootExpr?.isNone
+  checkTrue "(4_2 root)" $ ¬ state4_2.isSolved
   -- Ensure the proof can continue from `state4_2`.
   let state2b ← match state4_2.continue state2 with
     | .error msg => do
       addTest $ assertUnreachable $ msg
       return ()
     | .ok state => pure state
-  addTest $ LSpec.test "(resume)" (state2b.goals == [state2.goals.get! 0])
+  addTest $ LSpec.test "(resume)" (state2b.goals == [state2.goals[0]!])
   let state3_1 ← match ← state2b.tacticOn (goalId := 0) (tactic := "apply Or.inr") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   addTest $ LSpec.check "· apply Or.inr" (state3_1.goals.length = 1)
   let state4_1 ← match ← state3_1.tacticOn (goalId := 0) (tactic := "assumption") with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
   addTest $ LSpec.check "  assumption" state4_1.goals.isEmpty
-  addTest $ LSpec.check "(4_1 root)" state4_1.rootExpr?.isSome
+  checkTrue "(4_1 root)" $ ¬ state4_1.rootExpr?.get!.hasExprMVar
 
   return ()
   where
@@ -356,366 +303,6 @@ def test_or_comm: TestM Unit := do
       ]
     }
 
-example : ∀ (a b c1 c2: Nat), (b + a) + c1 = (b + a) + c2 → (a + b) + c1 = (b + a) + c2 := by
-  intro a b c1 c2 h
-  conv =>
-    lhs
-    congr
-    . rw [Nat.add_comm]
-    . rfl
-  exact h
-
-def test_conv: TestM Unit := do
-  let state? ← startProof (.expr "∀ (a b c1 c2: Nat), (b + a) + c1 = (b + a) + c2 → (a + b) + c1 = (b + a) + c2")
-  let state0 ← match state? with
-    | .some state => pure state
-    | .none => do
-      addTest $ assertUnreachable "Goal could not parse"
-      return ()
-
-  let tactic := "intro a b c1 c2 h"
-  let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check tactic ((← state1.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[interiorGoal [] "a + b + c1 = b + a + c2"])
-
-  let state2 ← match ← state1.conv (state1.get! 0) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check "conv => ..." ((← state2.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[{ interiorGoal [] "a + b + c1 = b + a + c2" with isConversion := true }])
-
-  let convTactic := "rhs"
-  let state3R ← match ← state2.tacticOn (goalId := 0) convTactic with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check s!"  {convTactic} (discard)" ((← state3R.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[{ interiorGoal [] "b + a + c2" with isConversion := true }])
-
-  let convTactic := "lhs"
-  let state3L ← match ← state2.tacticOn (goalId := 0) convTactic with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check s!"  {convTactic}" ((← state3L.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[{ interiorGoal [] "a + b + c1" with isConversion := true }])
-
-  let convTactic := "congr"
-  let state4 ← match ← state3L.tacticOn (goalId := 0) convTactic with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check s!"  {convTactic}" ((← state4.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[
-      { interiorGoal [] "a + b" with isConversion := true, userName? := .some "a" },
-      { interiorGoal [] "c1" with isConversion := true, userName? := .some "a" }
-    ])
-
-  let convTactic := "rw [Nat.add_comm]"
-  let state5_1 ← match ← state4.tacticOn (goalId := 0) convTactic with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check s!"  · {convTactic}" ((← state5_1.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[{ interiorGoal [] "b + a" with isConversion := true, userName? := .some "a" }])
-
-  let convTactic := "rfl"
-  let state6_1 ← match ← state5_1.tacticOn (goalId := 0) convTactic with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check s!"    {convTactic}" ((← state6_1.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[])
-
-  let state4_1 ← match state6_1.continue state4 with
-    | .ok state => pure state
-    | .error e => do
-      addTest $ expectationFailure "continue" e
-      return ()
-
-  let convTactic := "rfl"
-  let state6 ← match ← state4_1.tacticOn (goalId := 0) convTactic with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check s!"  · {convTactic}" ((← state6.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[])
-
-  let state1_1 ← match ← state6.convExit with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-
-  let tactic := "exact h"
-  let stateF ← match ← state1_1.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check tactic ((← stateF.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[])
-
-  where
-    h := "b + a + c1 = b + a + c2"
-    interiorGoal (free: List (String × String)) (target: String) :=
-      let free := [("a", "Nat"), ("b", "Nat"), ("c1", "Nat"), ("c2", "Nat"), ("h", h)] ++ free
-      buildGoal free target
-
-example : ∀ (a b c d: Nat), a + b = b + c → b + c = c + d → a + b = c + d := by
-  intro a b c d h1 h2
-  calc a + b = b + c := by apply h1
-    _ = c + d := by apply h2
-
-def test_calc: TestM Unit := do
-  let state? ← startProof (.expr "∀ (a b c d: Nat), a + b = b + c → b + c = c + d → a + b = c + d")
-  let state0 ← match state? with
-    | .some state => pure state
-    | .none => do
-      addTest $ assertUnreachable "Goal could not parse"
-      return ()
-  let tactic := "intro a b c d h1 h2"
-  let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check tactic ((← state1.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[interiorGoal [] "a + b = c + d"])
-  let pred := "a + b = b + c"
-  let state2 ← match ← state1.tryCalc (state1.get! 0) (pred := pred) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check s!"calc {pred} := _" ((← state2.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[
-      interiorGoal [] "a + b = b + c" (.some "calc"),
-      interiorGoal [] "b + c = c + d"
-    ])
-  addTest $ LSpec.test "(2.0 prev rhs)" (state2.calcPrevRhsOf? (state2.get! 0) |>.isNone)
-  addTest $ LSpec.test "(2.1 prev rhs)" (state2.calcPrevRhsOf? (state2.get! 1) |>.isSome)
-
-  let tactic := "apply h1"
-  let state2m ← match ← state2.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  let state3 ← match state2m.continue state2 with
-    | .ok state => pure state
-    | .error e => do
-      addTest $ expectationFailure "continue" e
-      return ()
-  let pred := "_ = c + d"
-  let state4 ← match ← state3.tryCalc (state3.get! 0) (pred := pred) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check s!"calc {pred} := _" ((← state4.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[
-      interiorGoal [] "b + c = c + d" (.some "calc")
-    ])
-  addTest $ LSpec.test "(4.0 prev rhs)" (state4.calcPrevRhsOf? (state4.get! 0) |>.isNone)
-  let tactic := "apply h2"
-  let state4m ← match ← state4.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.test "(4m root)" state4m.rootExpr?.isSome
-  where
-    interiorGoal (free: List (String × String)) (target: String) (userName?: Option String := .none) :=
-      let free := [("a", "Nat"), ("b", "Nat"), ("c", "Nat"), ("d", "Nat"),
-        ("h1", "a + b = b + c"), ("h2", "b + c = c + d")] ++ free
-      buildGoal free target userName?
-
-def test_nat_zero_add: TestM Unit := do
-  let state? ← startProof (.expr "∀ (n: Nat), n + 0 = n")
-  let state0 ← match state? with
-    | .some state => pure state
-    | .none => do
-      addTest $ assertUnreachable "Goal could not parse"
-      return ()
-  let tactic := "intro n"
-  let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check tactic ((← state1.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[buildGoal [("n", "Nat")] "n + 0 = n"])
-  let recursor := "@Nat.brecOn"
-  let state2 ← match ← state1.tryMotivatedApply (state1.get! 0) (recursor := recursor) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  let [mvarMotive, mvarMajor, mvarInduct, mvarConduit] := state2.goals |
-      fail "Incorrect number of goals"
-  let .num _ major := mvarMajor.name | fail "Incorrect form of mvar id"
-  addTest $ LSpec.check s!"mapply {recursor}" ((← state2.serializeGoals (options := ← read)).map (·.devolatilizeVars) =
-    #[
-      buildNamedGoal mvarMotive.name.toString  [("n", "Nat")] "Nat → Prop" (.some "motive"),
-      buildNamedGoal mvarMajor.name.toString   [("n", "Nat")] "Nat",
-      buildNamedGoal mvarInduct.name.toString  [("n", "Nat")] "∀ (t : Nat), Nat.below t → ?motive t",
-      buildNamedGoal mvarConduit.name.toString [("n", "Nat")] s!"?motive ?m.{major} = (n + 0 = n)" (.some "conduit")
-    ])
-
-  let tactic := "exact n"
-  let state3b ← match ← state2.tacticOn (goalId := 1) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check tactic ((← state3b.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[])
-  let state2b ← match state3b.continue state2 with
-    | .ok state => pure state
-    | .error e => do
-      addTest $ assertUnreachable e
-      return ()
-  let tactic := "exact (λ x => x + 0 = x)"
-  let state3c ← match ← state2b.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check tactic ((← state3c.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[])
-  let state2c ← match state3c.continue state2b with
-    | .ok state => pure state
-    | .error e => do
-      addTest $ assertUnreachable e
-      return ()
-  let tactic := "intro t h"
-  let state3 ← match ← state2c.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check tactic ((← state3.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[buildGoal [("n", "Nat"), ("t", "Nat"), ("h", "Nat.below t")] "t + 0 = t"])
-
-  let tactic := "simp"
-  let state3d ← match ← state3.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  let state2d ← match state3d.continue state2c with
-    | .ok state => pure state
-    | .error e => do
-      addTest $ assertUnreachable e
-      return ()
-  let tactic := "rfl"
-  let stateF ← match ← state2d.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check tactic ((← stateF.serializeGoals (options := ← read)) =
-    #[])
-
-  let expr := stateF.mctx.eAssignment.find! stateF.root
-  let (expr, _) := instantiateMVarsCore (mctx := stateF.mctx) (e := expr)
-  addTest $ LSpec.check "(F root)" stateF.rootExpr?.isSome
-
-def test_nat_zero_add_alt: TestM Unit := do
-  let state? ← startProof (.expr "∀ (n: Nat), n + 0 = n")
-  let state0 ← match state? with
-    | .some state => pure state
-    | .none => do
-      addTest $ assertUnreachable "Goal could not parse"
-      return ()
-  let tactic := "intro n"
-  let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check tactic ((← state1.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[buildGoal [("n", "Nat")] "n + 0 = n"])
-  let recursor := "@Nat.brecOn"
-  let state2 ← match ← state1.tryMotivatedApply (state1.get! 0) (recursor := recursor) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  let [mvarMotive, mvarMajor, mvarInduct, mvarConduit] := state2.goals |
-      fail "Incorrect number of goals"
-  let .num _ major := mvarMajor.name | fail "Incorrect form of mvar id"
-  addTest $ LSpec.check s!"mapply {recursor}" ((← state2.serializeGoals (options := ← read)).map (·.devolatilizeVars) =
-    #[
-      buildNamedGoal mvarMotive.name.toString [("n", "Nat")] "Nat → Prop" (.some "motive"),
-      buildNamedGoal mvarMajor.name.toString [("n", "Nat")] "Nat",
-      buildNamedGoal mvarInduct.name.toString [("n", "Nat")] "∀ (t : Nat), Nat.below t → ?motive t",
-      buildNamedGoal mvarConduit.name.toString [("n", "Nat")] s!"?motive ?m.{major} = (n + 0 = n)" (.some "conduit")
-    ])
-
-  let tactic := "intro x"
-  let state3m ← match ← state2.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  addTest $ LSpec.check tactic ((← state3m.serializeGoals (options := ← read)).map (·.devolatilize) =
-    #[buildGoal [("n", "Nat"), ("x", "Nat")] "Prop" (.some "motive")])
-  let tactic := "apply Eq"
-  let state3m2 ← match ← state3m.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
-    | other => do
-      addTest $ assertUnreachable $ other.toString
-      return ()
-  let [eqL, eqR, eqT] := state3m2.goals | fail "Incorrect number of goals"
-  let [_motive, _major, _step, conduit] := state2.goals | panic! "Goals conflict"
-  let state2b ← match state3m2.resume [conduit] with
-    | .ok state => pure state
-    | .error e => do
-      addTest $ assertUnreachable e
-      return ()
-
-  let cNatAdd := "(:c HAdd.hAdd) (:c Nat) (:c Nat) (:c Nat) ((:c instHAdd) (:c Nat) (:c instAddNat))"
-  let cNat0 := "((:c OfNat.ofNat) (:c Nat) (:lit 0) ((:c instOfNatNat) (:lit 0)))"
-  let fvN ← state2b.withContext conduit do
-    let lctx ← getLCtx
-    pure $ lctx.getFVarIds.get! 0 |>.name
-  let conduitRight := s!"((:c Eq) (:c Nat) ({cNatAdd} (:fv {fvN}) {cNat0}) (:fv {fvN}))"
-  let substOf (mvarId: MVarId) := s!"(:subst (:mv {mvarId.name}) (:fv {fvN}) (:mv {mvarMajor}))"
-  let .num _ nL := eqL.name | fail "Incorrect form of mvar id"
-  let .num _ nR := eqR.name | fail "Incorrect form of mvar id"
-  let nL' := nL + 4
-  let nR' := nR + 5
-  addTest $ LSpec.check "resume" ((← state2b.serializeGoals (options := { ← read with printExprAST := true })) =
-    #[
-      {
-        name := mvarConduit.name.toString,
-        userName? := .some "conduit",
-        target := {
-          pp? := .some s!"(?m.{nL'} ?m.{major} = ?m.{nR'} ?m.{major}) = (n + 0 = n)",
-          sexp? := .some s!"((:c Eq) (:sort 0) ((:c Eq) {substOf eqT} {substOf eqL} {substOf eqR}) {conduitRight})",
-        },
-        vars := #[{
-          name := fvN.toString,
-          userName := "n",
-          type? := .some { pp? := .some "Nat", sexp? := .some "(:c Nat)" },
-        }],
-      }
-    ])
-
 def test_tactic_failure_unresolved_goals : TestM Unit := do
   let state? ← startProof (.expr "∀ (p : Nat → Prop), ∃ (x : Nat), p (0 + x + 0)")
   let state0 ← match state? with
@@ -726,14 +313,16 @@ def test_tactic_failure_unresolved_goals : TestM Unit := do
 
   let tactic := "intro p"
   let state1 ← match ← state0.tacticOn 0 tactic with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
 
   let tactic := "exact ⟨0, by simp⟩"
-  let .failure messages ← state1.tacticOn 0 tactic | addTest $ assertUnreachable s!"{tactic} should fail"
-  checkEq s!"{tactic} fails" messages #[s!"{← getFileName}:0:12: error: unsolved goals\np : Nat → Prop\n⊢ p 0\n"]
+  let .failure #[message] ← state1.tacticOn 0 tactic
+    | fail s!"{tactic} should fail with 1 message"
+  checkEq s!"{tactic} fails" (← message.toString)
+    s!"{← getFileName}:0:12: error: unsolved goals\np : Nat → Prop\n⊢ p 0\n"
 
 def test_tactic_failure_synthesize_placeholder : TestM Unit := do
   let state? ← startProof (.expr "∀ (p q r : Prop) (h : p → q), q ∧ r")
@@ -745,7 +334,7 @@ def test_tactic_failure_synthesize_placeholder : TestM Unit := do
 
   let tactic := "intro p q r h"
   let state1 ← match ← state0.tacticOn 0 tactic with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -763,9 +352,29 @@ def test_tactic_failure_synthesize_placeholder : TestM Unit := do
   --  buildGoal [("p", "Prop"), ("q", "Prop"), ("r", "Prop"), ("h", "p → q")] "p ∧ r"
   --]
 
-  let .failure messages ← state1.tacticOn 0 tactic | addTest $ assertUnreachable s!"{tactic} should fail"
-  let message := s!"<Pantograph>:0:31: error: don't know how to synthesize placeholder\ncontext:\np q r : Prop\nh : p → q\n⊢ p ∧ r\n"
-  checkEq s!"{tactic} fails" messages #[message]
+  let .failure #[_head, message] ← state1.tacticOn 0 tactic
+    | addTest $ assertUnreachable s!"{tactic} should fail with 2 messages"
+  checkEq s!"{tactic} fails" (← message.toString)
+    s!"{← getFileName}:0:31: error: don't know how to synthesize placeholder\ncontext:\np q r : Prop\nh : p → q\n⊢ p ∧ r\n"
+
+def test_deconstruct : TestM Unit := do
+  let state? ← startProof (.expr "∀ (p q : Prop) (h : And p q), And q p")
+  let state0 ← match state? with
+    | .some state => pure state
+    | .none => do
+      addTest $ assertUnreachable "Goal could not parse"
+      return ()
+
+  let tactic := "intro p q ⟨hp, hq⟩"
+  let state1 ← match ← state0.tacticOn 0 tactic with
+    | .success state _ => pure state
+    | other => do
+      fail other.toString
+      return ()
+  checkEq tactic ((← state1.serializeGoals (options := ← read)).map (·.devolatilize))
+    #[
+      buildGoal [("p", "Prop"), ("q", "Prop"), ("hp", "p"), ("hq", "q")] "q ∧ p"
+    ]
 
 
 def suite (env: Environment): List (String × IO LSpec.TestSeq) :=
@@ -773,15 +382,11 @@ def suite (env: Environment): List (String × IO LSpec.TestSeq) :=
     ("identity", test_identity),
     ("Nat.add_comm", test_nat_add_comm false),
     ("Nat.add_comm manual", test_nat_add_comm true),
-    ("Nat.add_comm delta", test_delta_variable),
     ("arithmetic", test_arith),
     ("Or.comm", test_or_comm),
-    ("conv", test_conv),
-    ("calc", test_calc),
-    ("Nat.zero_add", test_nat_zero_add),
-    ("Nat.zero_add alt", test_nat_zero_add_alt),
     ("tactic failure with unresolved goals", test_tactic_failure_unresolved_goals),
     ("tactic failure with synthesize placeholder", test_tactic_failure_synthesize_placeholder),
+    ("deconstruct", test_deconstruct),
   ]
   tests.map (fun (name, test) => (name, proofRunner env test))
 

Test/Serial.lean

@@ -7,9 +7,6 @@ open Lean
 
 namespace Pantograph.Test.Serial
 
-def tempPath : IO System.FilePath := do
-  Prod.snd <$> IO.FS.createTempFile
-
 structure MultiState where
   coreContext : Core.Context
   env: Environment
@@ -30,29 +27,25 @@ def runCoreM { α } (state : Core.State) (testCoreM : TestT CoreM α) : TestM (
     set $ (← getThe LSpec.TestSeq) ++ tests
     return (a, state')
 
-def test_environment_pickling : TestM Unit := do
+def test_pickling_environment : TestM Unit := do
   let coreSrc : Core.State := { env := ← getEnv }
   let coreDst : Core.State := { env := ← getEnv }
 
   let name := `mystery
-  let envPicklePath ← tempPath
+  IO.FS.withTempFile λ _ envPicklePath => do
   let ((), _) ← runCoreM coreSrc do
     let type: Expr := .forallE `p (.sort 0) (.forallE `h (.bvar 0) (.bvar 1) .default) .default
     let value: Expr := .lam `p (.sort 0) (.lam `h (.bvar 0) (.bvar 0) .default) .default
-    let c := Lean.Declaration.defnDecl <| Lean.mkDefinitionValEx
+    let c := Declaration.defnDecl <| mkDefinitionValEx
       (name := name)
       (levelParams := [])
       (type := type)
       (value := value)
-      (hints := Lean.mkReducibilityHintsRegularEx 1)
-      (safety := Lean.DefinitionSafety.safe)
+      (hints := mkReducibilityHintsRegularEx 1)
+      (safety := .safe)
       (all := [])
-    let env' ← match (← getEnv).addDecl (← getOptions) c with
-      | .error e => do
-        let error ← (e.toMessageData (← getOptions)).toString
-        throwError error
-      | .ok env' => pure env'
-    environmentPickle env' envPicklePath
+    addDecl c
+    environmentPickle (← getEnv) envPicklePath
 
   let _ ← runCoreM coreDst do
     let (env', _) ← environmentUnpickle envPicklePath
@@ -60,13 +53,10 @@ def test_environment_pickling : TestM Unit := do
     let anotherName := `mystery2
     checkTrue s!"Doesn't have symbol {anotherName}" (env'.find? anotherName).isNone
 
-  IO.FS.removeFile envPicklePath
-
-def test_goal_state_pickling_simple : TestM Unit := do
+def test_goal_state_simple : TestM Unit := do
   let coreSrc : Core.State := { env := ← getEnv }
   let coreDst : Core.State := { env := ← getEnv }
-  let statePath ← tempPath
-
+  IO.FS.withTempFile λ _ statePath => do
   let type: Expr := .forallE `p (.sort 0) (.forallE `h (.bvar 0) (.bvar 1) .default) .default
   let stateGenerate : MetaM GoalState := runTermElabMInMeta do
     GoalState.create type
@@ -82,13 +72,50 @@ def test_goal_state_pickling_simple : TestM Unit := do
     let types ← metaM.run'
     checkTrue "Goals" $ types[0]!.equal type
 
-  IO.FS.removeFile statePath
+def test_pickling_env_extensions : TestM Unit := do
+  let coreSrc : Core.State := { env := ← getEnv }
+  let coreDst : Core.State := { env := ← getEnv }
+  IO.FS.withTempFile λ _ statePath => do
+  let ((), _) ← runCoreM coreSrc $ transformTestT runTermElabMInCore do
+    let .ok e ← elabTerm (← `(term|(2: Nat) ≤ 3 ∧ (3: Nat) ≤ 5)) .none | unreachable!
+    let state ← GoalState.create e
+    let .success state _ ← state.tacticOn' 0 (← `(tactic|apply And.intro)) | unreachable!
+
+    let goal := state.goals[0]!
+    let type ← goal.withContext do
+      let .ok type ← elabTerm (← `(term|(2: Nat) ≤ 3)) (.some $ .sort 0) | unreachable!
+      instantiateMVars type
+    let .success state1 _ ← state.tryTacticM goal (Tactic.assignWithAuxLemma type) | unreachable!
+    let parentExpr := state1.parentExpr!
+    checkTrue "src has aux lemma" $ parentExpr.getUsedConstants.any isAuxLemma
+    goalStatePickle state1 statePath
+  let ((), _) ← runCoreM coreDst $ transformTestT runTermElabMInCore do
+    let (state1, _) ← goalStateUnpickle statePath (← getEnv)
+    let parentExpr := state1.parentExpr!
+    checkTrue "dst has aux lemma" $ parentExpr.getUsedConstants.any isAuxLemma
+
+  return ()
+
+/-- Synthetic mvars in this case creates closures. These cannot be pickled. -/
+def test_pickling_synthetic_mvars : TestM Unit := do
+  let coreSrc : Core.State := { env := ← getEnv }
+  IO.FS.withTempFile λ _ statePath => do
+  let stateGenerate : MetaM GoalState := runTermElabMInMeta do
+    let type ← Elab.Term.elabTerm (← `(term|(0 : Nat) < 1)) .none
+    let state ← GoalState.create type
+    let .success state _ ← state.tryHave .unfocus "h" "0 < 2" | unreachable!
+    assert! state.savedState.term.elab.syntheticMVars.size > 0
+    return state
+
+  let ((), _) ← runCoreM coreSrc do
+    let state ← stateGenerate.run'
+    goalStatePickle state statePath
 
 structure Test where
   name : String
   routine: TestM Unit
 
-protected def Test.run (test: Test) (env: Lean.Environment) : IO LSpec.TestSeq := do
+protected def Test.run (test: Test) (env: Environment) : IO LSpec.TestSeq := do
   -- Create the state
   let state : MultiState := {
     coreContext := ← createCoreContext #[],
@@ -99,10 +126,12 @@ protected def Test.run (test: Test) (env: Lean.Environment) : IO LSpec.TestSeq :
   | .error e =>
     return LSpec.check s!"Emitted exception: {e.toString}" (e.toString == "")
 
-def suite (env : Lean.Environment): List (String × IO LSpec.TestSeq) :=
+def suite (env : Environment): List (String × IO LSpec.TestSeq) :=
   let tests: List Test := [
-    { name := "environment_pickling", routine := test_environment_pickling, },
-    { name := "goal_state_pickling_simple", routine := test_goal_state_pickling_simple, },
+    { name := "environment", routine := test_pickling_environment, },
+    { name := "goal simple", routine := test_goal_state_simple, },
+    { name := "goal synthetic mvars", routine := test_pickling_synthetic_mvars, },
+    { name := "extensions", routine := test_pickling_env_extensions, },
   ]
   tests.map (fun test => (test.name, test.run env))
 

Test/Tactic.lean

@@ -1,5 +1,4 @@
 import Test.Tactic.Assign
-import Test.Tactic.Congruence
-import Test.Tactic.MotivatedApply
-import Test.Tactic.NoConfuse
+import Test.Tactic.Fragment
 import Test.Tactic.Prograde
+import Test.Tactic.Special

Test/Tactic/Congruence.lean

@@ -1,88 +0,0 @@
-import LSpec
-import Lean
-import Test.Common
-
-open Lean
-open Pantograph
-
-namespace Pantograph.Test.Tactic.Congruence
-
-def test_congr_arg_list : TestT Elab.TermElabM Unit := do
-  let expr := "λ {α} (l1 l2 : List α) (h: l1 = l2) => l1.reverse = l2.reverse"
-  let expr ← parseSentence expr
-  Meta.lambdaTelescope expr $ λ _ body => do
-    let target ← Meta.mkFreshExprSyntheticOpaqueMVar body
-    let newGoals ← runTacticOnMVar Tactic.evalCongruenceArg target.mvarId!
-    addTest $ LSpec.check "goals" ((← newGoals.mapM (λ x => mvarUserNameAndType x)) =
-      [
-        (`α, "Sort ?u.30"),
-        (`a₁, "?α"),
-        (`a₂, "?α"),
-        (`f, "?α → List α"),
-        (`h, "?a₁ = ?a₂"),
-        (`conduit, "(?f ?a₁ = ?f ?a₂) = (l1.reverse = l2.reverse)"),
-      ])
-    let f := newGoals.get! 3
-    let h := newGoals.get! 4
-    let c := newGoals.get! 5
-    let results ← Meta.withAssignableSyntheticOpaque do f.apply (← parseSentence "List.reverse")
-    addTest $ LSpec.check "apply" (results.length = 0)
-    addTest $ LSpec.check "h" ((← exprToStr $ ← h.getType) = "?a₁ = ?a₂")
-    addTest $ LSpec.check "conduit" ((← exprToStr $ ← c.getType) = "(List.reverse ?a₁ = List.reverse ?a₂) = (l1.reverse = l2.reverse)")
-def test_congr_arg : TestT Elab.TermElabM Unit := do
-  let expr := "λ (n m: Nat) (h: n = m) => n * n = m * m"
-  let expr ← parseSentence expr
-  Meta.lambdaTelescope expr $ λ _ body => do
-    let target ← Meta.mkFreshExprSyntheticOpaqueMVar body
-    let newGoals ← runTacticOnMVar Tactic.evalCongruenceArg target.mvarId!
-    addTest $  LSpec.check "goals" ((← newGoals.mapM (λ x => mvarUserNameAndType x)) =
-      [
-        (`α, "Sort ?u.73"),
-        (`a₁, "?α"),
-        (`a₂, "?α"),
-        (`f, "?α → Nat"),
-        (`h, "?a₁ = ?a₂"),
-        (`conduit, "(?f ?a₁ = ?f ?a₂) = (n * n = m * m)"),
-      ])
-def test_congr_fun : TestT Elab.TermElabM Unit := do
-  let expr := "λ (n m: Nat) => (n + m) + (n + m) = (n + m) * 2"
-  let expr ← parseSentence expr
-  Meta.lambdaTelescope expr $ λ _ body => do
-    let target ← Meta.mkFreshExprSyntheticOpaqueMVar body
-    let newGoals ← runTacticOnMVar Tactic.evalCongruenceFun target.mvarId!
-    addTest $ LSpec.check "goals" ((← newGoals.mapM (λ x => mvarUserNameAndType x)) =
-      [
-        (`α, "Sort ?u.165"),
-        (`f₁, "?α → Nat"),
-        (`f₂, "?α → Nat"),
-        (`h, "?f₁ = ?f₂"),
-        (`a, "?α"),
-        (`conduit, "(?f₁ ?a = ?f₂ ?a) = (n + m + (n + m) = (n + m) * 2)"),
-      ])
-def test_congr : TestT Elab.TermElabM Unit := do
-  let expr := "λ (a b: Nat) => a = b"
-  let expr ← parseSentence expr
-  Meta.lambdaTelescope expr $ λ _ body => do
-    let target ← Meta.mkFreshExprSyntheticOpaqueMVar body
-    let newGoals ← runTacticOnMVar Tactic.evalCongruence target.mvarId!
-    addTest $  LSpec.check "goals" ((← newGoals.mapM (λ x => mvarUserNameAndType x)) =
-      [
-        (`α, "Sort ?u.10"),
-        (`f₁, "?α → Nat"),
-        (`f₂, "?α → Nat"),
-        (`a₁, "?α"),
-        (`a₂, "?α"),
-        (`h₁, "?f₁ = ?f₂"),
-        (`h₂, "?a₁ = ?a₂"),
-        (`conduit, "(?f₁ ?a₁ = ?f₂ ?a₂) = (a = b)"),
-      ])
-
-def suite (env: Environment): List (String × IO LSpec.TestSeq) :=
-  [
-    ("congrArg List.reverse", test_congr_arg_list),
-    ("congrArg", test_congr_arg),
-    ("congrFun", test_congr_fun),
-    ("congr", test_congr),
-  ] |>.map (λ (name, t) => (name, runTestTermElabM env t))
-
-end Pantograph.Test.Tactic.Congruence

Test/Tactic/Fragment.lean

@@ -0,0 +1,313 @@
+import Pantograph.Goal
+import Test.Common
+
+open Lean
+
+namespace Pantograph.Test.Tactic.Fragment
+
+private def buildGoal (nameType: List (String × String)) (target: String):
+    Protocol.Goal :=
+  {
+    target := { pp? := .some target},
+    vars := (nameType.map fun x => ({
+      userName := x.fst,
+      type? := .some { pp? := .some x.snd },
+    })).toArray
+  }
+
+abbrev TestM := TestT $ Elab.TermElabM
+
+example : ∀ (a b c1 c2: Nat), (b + a) + c1 = (b + a) + c2 → (a + b) + c1 = (b + a) + c2 := by
+  intro a b c1 c2 h
+  conv =>
+    lhs
+    congr
+    . rw [Nat.add_comm]
+    . rfl
+  exact h
+
+def test_conv_simple: TestM Unit := do
+  let rootTarget ← parseSentence "∀ (a b c1 c2: Nat), (b + a) + c1 = (b + a) + c2 → (a + b) + c1 = (b + a) + c2"
+  let state0 ← GoalState.create rootTarget
+
+  let tactic := "intro a b c1 c2 h"
+  let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := tactic) with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  addTest $ LSpec.check tactic ((← state1.serializeGoals).map (·.devolatilize) =
+    #[interiorGoal [] "a + b + c1 = b + a + c2"])
+
+  let goalConv := state1.goals[0]!
+  let state2 ← match ← state1.convEnter (state1.get! 0) with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  addTest $ LSpec.check "conv => ..." ((← state2.serializeGoals).map (·.devolatilize) =
+    #[{ interiorGoal [] "a + b + c1 = b + a + c2" with fragment := .conv }])
+
+  let convTactic := "rhs"
+  let state3R ← match ← state2.tacticOn (goalId := 0) convTactic with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  addTest $ LSpec.check s!"  {convTactic} (discard)" ((← state3R.serializeGoals).map (·.devolatilize) =
+    #[{ interiorGoal [] "b + a + c2" with fragment := .conv }])
+
+  let convTactic := "lhs"
+  let state3L ← match ← state2.tacticOn (goalId := 0) convTactic with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  addTest $ LSpec.check s!"  {convTactic}" ((← state3L.serializeGoals).map (·.devolatilize) =
+    #[{ interiorGoal [] "a + b + c1" with fragment := .conv }])
+
+  let convTactic := "congr"
+  let state4 ← match ← state3L.tacticOn (goalId := 0) convTactic with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  addTest $ LSpec.check s!"  {convTactic}" ((← state4.serializeGoals).map (·.devolatilize) =
+    #[
+      { interiorGoal [] "a + b" with fragment := .conv, userName? := .some "a" },
+      { interiorGoal [] "c1" with fragment := .conv, userName? := .some "a" }
+    ])
+
+  let convTactic := "rw [Nat.add_comm]"
+  let state5_1 ← match ← state4.tacticOn (goalId := 0) convTactic with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  addTest $ LSpec.check s!"  · {convTactic}" ((← state5_1.serializeGoals).map (·.devolatilize) =
+    #[{ interiorGoal [] "b + a" with fragment := .conv, userName? := .some "a" }])
+
+  let convTactic := "rfl"
+  let state6_1 ← match ← state5_1.tacticOn (goalId := 0) convTactic with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  addTest $ LSpec.check s!"    {convTactic}" ((← state6_1.serializeGoals).map (·.devolatilize) =
+    #[])
+
+  let state4_1 ← match state6_1.continue state4 with
+    | .ok state => pure state
+    | .error e => do
+      addTest $ expectationFailure "continue" e
+      return ()
+
+  let convTactic := "rfl"
+  let state1_1 ← match ← state4_1.tacticOn (goalId := 0) convTactic with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  addTest $ LSpec.check s!"  · {convTactic}" ((← state1_1.serializeGoals).map (·.devolatilize) =
+    #[interiorGoal [] "b + a + c1 = b + a + c2"])
+  checkEq "(fragments)" state1_1.fragments.size 0
+
+  /-
+  let state1_1 ← match ← state6.fragmentExit goalConv with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  -/
+
+  let tactic := "exact h"
+  let stateF ← match ← state1_1.tacticOn (goalId := 0) (tactic := tactic) with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  checkEq tactic ((← stateF.serializeGoals).map (·.devolatilize)) #[]
+  checkEq "fragments" stateF.fragments.size 0
+
+  where
+    h := "b + a + c1 = b + a + c2"
+    interiorGoal (free: List (String × String)) (target: String) :=
+      let free := [("a", "Nat"), ("b", "Nat"), ("c1", "Nat"), ("c2", "Nat"), ("h", h)] ++ free
+      buildGoal free target
+
+example (p : Prop) (x y z : Nat) : p → (p → x = y) → x + z = y + z ∧ p := by
+  intro hp hi
+  apply And.intro
+  conv =>
+    rhs
+    arg 1
+    rw [←hi]
+    rfl
+    tactic => exact hp
+  exact hp
+
+def test_conv_unshielded : TestM Unit := do
+  let rootTarget ← parseSentence "∀ (p : Prop) (x y z : Nat), p → (p → x = y) → x + z = y + z ∧ p"
+  let state ← GoalState.create rootTarget
+  let tactic := "intro p x y z hp hi"
+  let .success state _ ← state.tacticOn 0 tactic | fail "intro failed"
+  let tactic := "apply And.intro"
+  let .success state _ ← state.tacticOn 0 tactic | fail "apply failed"
+  let .success state _ ← state.convEnter (.prefer state.goals[0]!) | fail "Cannot enter conversion tactic mode"
+  let .success state _ ← state.tryTactic .unfocus "rhs" | fail "rhs failed"
+  let tactic := "arg 1"
+  let .success state _ ← state.tryTactic .unfocus tactic | fail s!"{tactic} failed"
+  checkEq s!"  {tactic}" ((← state.serializeGoals).map (·.devolatilize))
+    #[
+      { interiorGoal [] "y" with fragment := .conv },
+      { interiorGoal [] "p" with userName? := "right", },
+    ]
+  let tactic := "rw [←hi]"
+  let .success state _ ← state.tryTactic .unfocus tactic | fail s!"{tactic} failed"
+  checkEq s!"  {tactic}" state.goals.length 3
+  let tactic := "rfl"
+  let .success state _ ← state.tryTactic .unfocus tactic | fail s!"{tactic} failed"
+  checkEq s!"  {tactic}" ((← state.serializeGoals).map (·.devolatilize))
+    #[
+      interiorGoal [] "p",
+      { interiorGoal [] "p" with userName? := "right", },
+    ]
+  checkEq "(n goals)" state.goals.length 2
+  checkEq "(fragments)" state.fragments.size 0
+  let tactic := "exact hp"
+  let .success state _ ← state.tryTactic .unfocus tactic | fail s!"{tactic} failed"
+  let tactic := "exact hp"
+  let .success state _ ← state.tryTactic .unfocus tactic | fail s!"{tactic} failed"
+  let root? := state.rootExpr?
+  checkTrue "root" root?.isSome
+  checkEq "fragments" state.fragments.size 0
+  where
+  interiorGoal (free: List (String × String)) (target: String) :=
+    let free := [("p", "Prop"), ("x", "Nat"), ("y", "Nat"), ("z", "Nat"), ("hp", "p"), ("hi", "p → x = y")] ++ free
+    buildGoal free target
+
+example : ∀ (x y z w : Nat), y = z → x + z = w → x + y = w := by
+  intro x y z w hyz hxzw
+  conv =>
+    lhs
+    arg 2
+    rw [hyz]
+    rfl
+  exact hxzw
+
+def test_conv_unfinished : TestM Unit := do
+  let rootTarget ← parseSentence "∀ (x y z w : Nat), y = z → x + z = w → x + y = w"
+  let state ← GoalState.create rootTarget
+  let tactic := "intro x y z w hyz hxzw"
+  let .success state _ ← state.tacticOn 0 tactic | fail "intro failed"
+  let convParent := state.goals[0]!
+  let .success state _ ← state.convEnter (.prefer convParent) | fail "Cannot enter conversion tactic mode"
+  let .success state _ ← state.tryTactic .unfocus "lhs" | fail "rhs failed"
+  let tactic := "arg 2"
+  let .success state _ ← state.tryTactic .unfocus tactic | fail s!"{tactic} failed"
+  checkEq s!"  {tactic}" ((← state.serializeGoals).map (·.devolatilize))
+    #[
+      { interiorGoal [] "y" with fragment := .conv },
+    ]
+  let tactic := "rw [hyz]"
+  let .success state _ ← state.tryTactic .unfocus tactic | fail s!"{tactic} failed"
+  checkEq s!"  {tactic}" ((← state.serializeGoals).map (·.devolatilize))
+    #[
+      { interiorGoal [] "z" with fragment := .conv },
+    ]
+  checkTrue "  (fragment)" $ state.fragments.contains state.mainGoal?.get!
+  checkTrue "  (fragment parent)" $ state.fragments.contains convParent
+  checkTrue "  (main goal)" state.mainGoal?.isSome
+  let tactic := "rfl"
+  let .success state _ ← state.tryTactic .unfocus tactic | fail s!"{tactic} failed"
+  checkEq s!"  {tactic}" ((← state.serializeGoals).map (·.devolatilize))
+    #[
+      interiorGoal [] "x + z = w",
+    ]
+  checkEq "(fragments)" state.fragments.size 0
+  checkEq s!"  {tactic}" state.goals.length 1
+  let tactic := "exact hxzw"
+  let .success state _ ← state.tryTactic .unfocus tactic | fail s!"{tactic} failed"
+  let root? := state.rootExpr?
+  checkTrue "root" root?.isSome
+  where
+  interiorGoal (free: List (String × String)) (target: String) :=
+    let free := [("x", "Nat"), ("y", "Nat"), ("z", "Nat"), ("w", "Nat"), ("hyz", "y = z"), ("hxzw", "x + z = w")] ++ free
+    buildGoal free target
+
+example : ∀ (a b c d: Nat), a + b = b + c → b + c = c + d → a + b = c + d := by
+  intro a b c d h1 h2
+  calc a + b = b + c := by apply h1
+    _ = c + d := by apply h2
+
+def test_calc: TestM Unit := do
+  let rootTarget ← parseSentence "∀ (a b c d: Nat), a + b = b + c → b + c = c + d → a + b = c + d"
+  let state0 ← GoalState.create rootTarget
+  let tactic := "intro a b c d h1 h2"
+  let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := tactic) with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  addTest $ LSpec.check tactic ((← state1.serializeGoals).map (·.devolatilize) =
+    #[interiorGoal [] "a + b = c + d"])
+  let pred := "a + b = b + c"
+  let .success state1 _ ← state1.calcEnter state1.mainGoal?.get! | fail "Could not enter calc"
+  let state2 ← match ← state1.tacticOn 0 pred with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  addTest $ LSpec.check s!"calc {pred} := _" ((← state2.serializeGoals).map (·.devolatilize) =
+    #[
+      { interiorGoal [] "a + b = b + c" with userName? := .some "calc" },
+      { interiorGoal [] "b + c = c + d" with fragment := .calc },
+    ])
+  addTest $ LSpec.test "(2.0 prev rhs)" (state2.calcPrevRhsOf? (state2.get! 0) |>.isNone)
+  addTest $ LSpec.test "(2.1 prev rhs)" (state2.calcPrevRhsOf? (state2.get! 1) |>.isSome)
+
+  let tactic := "apply h1"
+  let state2m ← match ← state2.tacticOn (goalId := 0) (tactic := tactic) with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  let state3 ← match state2m.continue state2 with
+    | .ok state => pure state
+    | .error e => do
+      addTest $ expectationFailure "continue" e
+      return ()
+  let pred := "_ = c + d"
+  let state4 ← match ← state3.tacticOn 0 pred with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  addTest $ LSpec.check s!"calc {pred} := _" ((← state4.serializeGoals).map (·.devolatilize) =
+    #[
+      { interiorGoal [] "b + c = c + d" with userName? := .some "calc" },
+    ])
+  addTest $ LSpec.test "(4.0 prev rhs)" (state4.calcPrevRhsOf? (state4.get! 0) |>.isNone)
+  let tactic := "apply h2"
+  let state4m ← match ← state4.tacticOn (goalId := 0) (tactic := tactic) with
+    | .success state _ => pure state
+    | other => do
+      addTest $ assertUnreachable $ other.toString
+      return ()
+  checkEq "(fragments)" state4m.fragments.size 0
+  addTest $ LSpec.test "(4m root)" state4m.rootExpr?.isSome
+  where
+    interiorGoal (free: List (String × String)) (target: String) :=
+      let free := [("a", "Nat"), ("b", "Nat"), ("c", "Nat"), ("d", "Nat"),
+        ("h1", "a + b = b + c"), ("h2", "b + c = c + d")] ++ free
+      buildGoal free target
+
+def suite (env: Environment): List (String × IO LSpec.TestSeq) :=
+  [
+    ("conv simple", test_conv_simple),
+    ("conv unshielded", test_conv_unshielded),
+    ("conv unfinished", test_conv_unfinished),
+    ("calc", test_calc),
+  ] |>.map (λ (name, t) => (name, runTestTermElabM env t))
+
+end Pantograph.Test.Tactic.Fragment

Test/Tactic/MotivatedApply.lean

@@ -1,113 +0,0 @@
-import LSpec
-import Lean
-import Test.Common
-
-open Lean
-open Pantograph
-
-namespace Pantograph.Test.Tactic.MotivatedApply
-
-def test_type_extract : TestT Elab.TermElabM Unit := do
-  let recursor ← parseSentence "@Nat.brecOn"
-  let recursorType ← Meta.inferType recursor
-  addTest $ LSpec.check "recursorType" ("{motive : Nat → Sort ?u.1} → (t : Nat) → ((t : Nat) → Nat.below t → motive t) → motive t" =
-    (← exprToStr recursorType))
-  let info ← match Tactic.getRecursorInformation recursorType with
-    | .some info => pure info
-    | .none => throwError "Failed to extract recursor info"
-  addTest $ LSpec.check "iMotive" (info.iMotive = 2)
-  let motiveType := info.getMotiveType
-  addTest $ LSpec.check "motiveType" ("Nat → Sort ?u.1" =
-    (← exprToStr motiveType))
-
-def test_nat_brec_on : TestT Elab.TermElabM Unit := do
-  let expr := "λ (n t: Nat) => n + 0 = n"
-  let expr ← parseSentence expr
-  Meta.lambdaTelescope expr $ λ _ body => do
-    let recursor ← match Parser.runParserCategory
-      (env := ← MonadEnv.getEnv)
-      (catName := `term)
-      (input := "@Nat.brecOn")
-      (fileName := ← getFileName) with
-      | .ok syn => pure syn
-      | .error error => throwError "Failed to parse: {error}"
-    -- Apply the tactic
-    let target ← Meta.mkFreshExprSyntheticOpaqueMVar body
-    let tactic := Tactic.evalMotivatedApply recursor
-    let newGoals ← runTacticOnMVar tactic target.mvarId!
-    let test :=  LSpec.check "goals" ((← newGoals.mapM (λ g => do exprToStr (← g.getType))) =
-      [
-        "Nat → Prop",
-        "Nat",
-        "∀ (t : Nat), Nat.below t → ?motive t",
-        "?motive ?m.74 = (n + 0 = n)",
-      ])
-    addTest test
-
-def test_list_brec_on : TestT Elab.TermElabM Unit := do
-  let expr := "λ {α : Type} (l: List α) => l ++ [] = [] ++ l"
-  let expr ← parseSentence expr
-  Meta.lambdaTelescope expr $ λ _ body => do
-    let recursor ← match Parser.runParserCategory
-      (env := ← MonadEnv.getEnv)
-      (catName := `term)
-      (input := "@List.brecOn")
-      (fileName := ← getFileName) with
-      | .ok syn => pure syn
-      | .error error => throwError "Failed to parse: {error}"
-    -- Apply the tactic
-    let target ← Meta.mkFreshExprSyntheticOpaqueMVar body
-    let tactic := Tactic.evalMotivatedApply recursor
-    let newGoals ← runTacticOnMVar tactic target.mvarId!
-    addTest $ LSpec.check "goals" ((← newGoals.mapM (λ g => do exprToStr (← g.getType))) =
-      [
-        "Type ?u.90",
-        "List ?m.92 → Prop",
-        "List ?m.92",
-        "∀ (t : List ?m.92), List.below t → ?motive t",
-        "?motive ?m.94 = (l ++ [] = [] ++ l)",
-      ])
-
-def test_partial_motive_instantiation : TestT Elab.TermElabM Unit := do
-  let expr := "λ (n t: Nat) => n + 0 = n"
-  let recursor ← match Parser.runParserCategory
-    (env := ← MonadEnv.getEnv)
-    (catName := `term)
-    (input := "@Nat.brecOn")
-    (fileName := ← getFileName) with
-    | .ok syn => pure syn
-    | .error error => throwError "Failed to parse: {error}"
-  let expr ← parseSentence expr
-  Meta.lambdaTelescope expr $ λ _ body => do
-    -- Apply the tactic
-    let target ← Meta.mkFreshExprSyntheticOpaqueMVar body
-    let tactic := Tactic.evalMotivatedApply recursor
-    let newGoals ← runTacticOnMVar tactic target.mvarId!
-    let majorId := 74
-    addTest $ (LSpec.check "goals" ((← newGoals.mapM (λ g => do exprToStr (← g.getType))) =
-      [
-        "Nat → Prop",
-        "Nat",
-        "∀ (t : Nat), Nat.below t → ?motive t",
-        s!"?motive ?m.{majorId} = (n + 0 = n)",
-      ]))
-    let [motive, major, step, conduit] := newGoals | panic! "Incorrect goal number"
-    addTest $ (LSpec.check "goal name" (major.name.toString = s!"_uniq.{majorId}"))
-
-    -- Assign motive to `λ x => x + _`
-    let motive_assign ← parseSentence "λ (x: Nat) => @Nat.add x + 0 = _"
-    motive.assign motive_assign
-
-    addTest $ ← conduit.withContext do
-      let t := toString (← Meta.ppExpr $ ← conduit.getType)
-      return LSpec.check "conduit" (t = s!"(Nat.add ?m.{majorId} + 0 = ?m.149 ?m.{majorId}) = (n + 0 = n)")
-
-def suite (env: Environment): List (String × IO LSpec.TestSeq) :=
-  [
-    ("type_extract", test_type_extract),
-    ("Nat.brecOn", test_nat_brec_on),
-    ("List.brecOn", test_list_brec_on),
-    ("Nat.brecOn partial motive instantiation", test_partial_motive_instantiation),
-  ] |>.map (λ (name, t) => (name, runTestTermElabM env t))
-
-end Pantograph.Test.Tactic.MotivatedApply

Test/Tactic/NoConfuse.lean

@@ -1,72 +0,0 @@
-import LSpec
-import Lean
-import Test.Common
-
-open Lean
-open Pantograph
-
-namespace Pantograph.Test.Tactic.NoConfuse
-
-def test_nat : TestT Elab.TermElabM Unit := do
-  let expr := "λ (n: Nat) (h: 0 = n + 1) => False"
-  let expr ← parseSentence expr
-  Meta.lambdaTelescope expr $ λ _ body => do
-    let recursor ← match Parser.runParserCategory
-      (env := ← MonadEnv.getEnv)
-      (catName := `term)
-      (input := "h")
-      (fileName := ← getFileName) with
-      | .ok syn => pure syn
-      | .error error => throwError "Failed to parse: {error}"
-    -- Apply the tactic
-    let target ← Meta.mkFreshExprSyntheticOpaqueMVar body
-    let tactic := Tactic.evalNoConfuse recursor
-    let newGoals ← runTacticOnMVar tactic target.mvarId!
-    addTest $ LSpec.check "goals" ((← newGoals.mapM (λ g => do exprToStr (← g.getType))) = [])
-
-def test_nat_fail : TestT Elab.TermElabM Unit := do
-  let expr := "λ (n: Nat) (h: n = n) => False"
-  let expr ← parseSentence expr
-  Meta.lambdaTelescope expr $ λ _ body => do
-    let recursor ← match Parser.runParserCategory
-      (env := ← MonadEnv.getEnv)
-      (catName := `term)
-      (input := "h")
-      (fileName := ← getFileName) with
-      | .ok syn => pure syn
-      | .error error => throwError "Failed to parse: {error}"
-    -- Apply the tactic
-    let target ← Meta.mkFreshExprSyntheticOpaqueMVar body
-    try
-      let tactic := Tactic.evalNoConfuse recursor
-      let _ ← runTacticOnMVar tactic target.mvarId!
-      addTest $ assertUnreachable "Tactic should fail"
-    catch _ =>
-      addTest $ LSpec.check "Tactic should fail" true
-
-def test_list : TestT Elab.TermElabM Unit := do
-  let expr := "λ (l: List Nat) (h: [] = 1 :: l) => False"
-  let expr ← parseSentence expr
-  Meta.lambdaTelescope expr $ λ _ body => do
-    let recursor ← match Parser.runParserCategory
-      (env := ← MonadEnv.getEnv)
-      (catName := `term)
-      (input := "h")
-      (fileName := ← getFileName) with
-      | .ok syn => pure syn
-      | .error error => throwError "Failed to parse: {error}"
-    -- Apply the tactic
-    let target ← Meta.mkFreshExprSyntheticOpaqueMVar body
-    let tactic := Tactic.evalNoConfuse recursor
-    let newGoals ← runTacticOnMVar tactic target.mvarId!
-    addTest $ LSpec.check "goals"
-      ((← newGoals.mapM (λ g => do exprToStr (← g.getType))) = [])
-
-def suite (env: Environment): List (String × IO LSpec.TestSeq) :=
-  [
-    ("Nat", test_nat),
-    ("Nat fail", test_nat_fail),
-    ("List", test_list),
-  ] |>.map (λ (name, t) => (name, runTestTermElabM env t))
-
-end Pantograph.Test.Tactic.NoConfuse

Test/Tactic/Prograde.lean

@@ -56,7 +56,7 @@ def test_define_proof : TestT Elab.TermElabM Unit := do
   let state0 ← GoalState.create rootExpr
   let tactic := "intro p q h"
   let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -65,7 +65,7 @@ def test_define_proof : TestT Elab.TermElabM Unit := do
 
   let expr := "Or.inl (Or.inl h)"
   let state2 ← match ← state1.tryAssign (state1.get! 0) (expr := expr) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -75,7 +75,7 @@ def test_define_proof : TestT Elab.TermElabM Unit := do
   let evalBind := "y"
   let evalExpr := "Or.inl h"
   let state2 ← match ← state1.tryDefine (state1.get! 0) (binderName := evalBind) (expr := evalExpr) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -87,7 +87,7 @@ def test_define_proof : TestT Elab.TermElabM Unit := do
         { userName := "q", type? := .some { pp? := .some "Prop" } },
         { userName := "h", type? := .some { pp? := .some "p" } },
         { userName := "y",
-          type? := .some { pp? := .some "p ∨ ?m.25" },
+          type? := .some { pp? := .some "p ∨ ?m.19" },
           value? := .some { pp? := .some "Or.inl h" },
         }
       ]
@@ -95,7 +95,7 @@ def test_define_proof : TestT Elab.TermElabM Unit := do
 
   let expr := "Or.inl y"
   let state3 ← match ← state2.tryAssign (state2.get! 0) (expr := expr) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -112,35 +112,31 @@ def fun_define_root_expr: ∀ (p: Prop), PProd (Nat → p) Unit → p := by
 def test_define_root_expr : TestT Elab.TermElabM Unit := do
   --let rootExpr ← parseSentence "Nat"
   --let state0 ← GoalState.create rootExpr
-  --let .success state1 ← state0.tacticOn (goalId := 0) "exact 5" | addTest $ assertUnreachable "exact 5"
+  --let .success state1 _ ← state0.tacticOn (goalId := 0) "exact 5" | addTest $ assertUnreachable "exact 5"
   --let .some rootExpr := state1.rootExpr? | addTest $ assertUnreachable "Root expr"
   --addTest $ LSpec.check "root" ((toString $ ← Meta.ppExpr rootExpr) = "5")
   let rootExpr ← parseSentence "∀ (p: Prop), PProd (Nat → p) Unit → p"
   let state0 ← GoalState.create rootExpr
   let tactic := "intro p x"
-  let .success state1 ← state0.tacticOn (goalId := 0) tactic | addTest $ assertUnreachable tactic
+  let .success state1 _ ← state0.tacticOn (goalId := 0) tactic | addTest $ assertUnreachable tactic
   let binderName := `binder
   let value := "x.fst"
   let expr ← state1.goals[0]!.withContext $ strToTermSyntax value
   let tacticM := Tactic.evalDefine binderName expr
-  let .success state2 ← state1.tryTacticM (state1.get! 0) tacticM | addTest $ assertUnreachable s!"define {binderName} := {value}"
+  let .success state2 _ ← state1.tryTacticM (state1.get! 0) tacticM | addTest $ assertUnreachable s!"define {binderName} := {value}"
   let tactic := s!"apply {binderName}"
-  let .success state3 ← state2.tacticOn (goalId := 0) tactic | addTest $ assertUnreachable tactic
+  let .success state3 _ ← state2.tacticOn (goalId := 0) tactic | addTest $ assertUnreachable tactic
   let tactic := s!"exact 5"
-  let .success state4 ← state3.tacticOn (goalId := 0) tactic | addTest $ assertUnreachable tactic
+  let .success state4 _ ← state3.tacticOn (goalId := 0) tactic | addTest $ assertUnreachable tactic
   let .some rootExpr := state4.rootExpr? | addTest $ assertUnreachable "Root expr"
   addTest $ LSpec.check "root" ((toString $ ← Meta.ppExpr rootExpr) = "fun p x =>\n  let binder := x.fst;\n  binder 5")
 
---set_option pp.all true
---#check @PSigma (α := Prop) (β := λ (p: Prop) => p)
---def test_define_root_expr : TestT Elab.TermElabM Unit := do
-
 def test_have_proof : TestT Elab.TermElabM Unit := do
   let rootExpr ← parseSentence "∀ (p q: Prop), p → ((p ∨ q) ∨ (p ∨ q))"
   let state0 ← GoalState.create rootExpr
   let tactic := "intro p q h"
   let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -149,7 +145,7 @@ def test_have_proof : TestT Elab.TermElabM Unit := do
 
   let expr := "Or.inl (Or.inl h)"
   let state2 ← match ← state1.tryAssign (state1.get! 0) (expr := expr) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -159,7 +155,7 @@ def test_have_proof : TestT Elab.TermElabM Unit := do
   let haveBind := "y"
   let haveType := "p ∨ q"
   let state2 ← match ← state1.tryHave (state1.get! 0) (binderName := haveBind) (type := haveType) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -171,7 +167,7 @@ def test_have_proof : TestT Elab.TermElabM Unit := do
 
   let expr := "Or.inl h"
   let state3 ← match ← state2.tryAssign (state2.get! 0) (expr := expr) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -185,12 +181,11 @@ def test_have_proof : TestT Elab.TermElabM Unit := do
       return ()
   let expr := "Or.inl y"
   let state4 ← match ← state2b.tryAssign (state2b.get! 0) (expr := expr) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
-  addTest $ LSpec.check s!":= {expr}" ((← state4.serializeGoals).map (·.devolatilize) =
-    #[])
+  checkEq s!":= {expr}" ((← state4.serializeGoals).map (·.devolatilize)) #[]
 
   let .some rootExpr := state4.rootExpr? | addTest $ assertUnreachable "Root expr"
   addTest $ LSpec.check "root" ((toString $ ← Meta.ppExpr rootExpr) = "fun p q h y => Or.inl y")
@@ -200,7 +195,7 @@ def test_let (specialized: Bool): TestT Elab.TermElabM Unit := do
   let state0 ← GoalState.create rootExpr
   let tactic := "intro a p h"
   let state1 ← match ← state0.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -216,7 +211,7 @@ def test_let (specialized: Bool): TestT Elab.TermElabM Unit := do
     | true => state1.tryLet (state1.get! 0) (binderName := "b") (type := letType)
     | false => state1.tryAssign (state1.get! 0) (expr := expr)
   let state2 ← match result2 with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -241,7 +236,7 @@ def test_let (specialized: Bool): TestT Elab.TermElabM Unit := do
 
   let tactic := "exact 1"
   let state3 ← match ← state2.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()
@@ -268,13 +263,15 @@ def test_let (specialized: Bool): TestT Elab.TermElabM Unit := do
   let tactic := "exact h"
   match ← state3r.tacticOn (goalId := 0) (tactic := tactic) with
   | .failure #[message] =>
-    addTest $ LSpec.check tactic  (message = s!"type mismatch\n  h\nhas type\n  a : Prop\nbut is expected to have type\n  {mainTarget} : Prop")
+    checkEq tactic
+      (← message.toString)
+      s!"{← getFileName}:0:0: error: type mismatch\n  h\nhas type\n  a : Prop\nbut is expected to have type\n  {mainTarget} : Prop\n"
   | other => do
-    addTest $ assertUnreachable $ other.toString
+    fail s!"Should be a failure: {other.toString}"
 
   let tactic := "exact Or.inl (Or.inl h)"
   let state4 ← match ← state3r.tacticOn (goalId := 0) (tactic := tactic) with
-    | .success state => pure state
+    | .success state _ => pure state
     | other => do
       addTest $ assertUnreachable $ other.toString
       return ()

Test/Tactic/Special.lean

@@ -0,0 +1,25 @@
+import LSpec
+import Lean
+import Test.Common
+
+open Lean
+open Pantograph
+
+namespace Pantograph.Test.Tactic.Special
+
+def test_exact_q : TestT Elab.TermElabM Unit := do
+  let rootExpr ← parseSentence "1 + 2 = 2 + 3"
+  let state0 ← GoalState.create rootExpr
+  let tactic := "exact?"
+  let state1? ← state0.tacticOn (goalId := 0) (tactic := tactic)
+  let .failure messages := state1? | fail "Must fail"
+  checkEq "messages"
+    (← messages.mapM (·.toString))
+    #[s!"{← getFileName}:0:0: error: `exact?` could not close the goal. Try `apply?` to see partial suggestions.\n"]
+
+def suite (env: Environment): List (String × IO LSpec.TestSeq) :=
+  [
+    ("exact?", test_exact_q),
+  ] |>.map (λ (name, t) => (name, runTestTermElabM env t))
+
+end Pantograph.Test.Tactic.Special

Tomograph.lean

@@ -0,0 +1,41 @@
+/- A tool for analysing Lean source code. -/
+import Pantograph.Frontend
+import Pantograph.Library
+
+open Lean
+
+namespace Pantograph
+
+def fail (s : String) : IO UInt32 := do
+  IO.eprintln s
+  return 2
+
+def dissect (args : List String) : IO UInt32 := do
+  let fileName :: _args := args | fail s!"Must supply a file name"
+  let file ← IO.FS.readFile fileName
+  let (context, state) ← do Frontend.createContextStateFromFile file fileName (env? := .none) {}
+  let frontendM: Frontend.FrontendM _ :=
+    Frontend.mapCompilationSteps λ step => do
+      IO.println s!"🐈 {step.stx.getKind.toString}"
+      for (tree, i) in step.trees.zipIdx do
+        IO.println s!"🌲[{i}] {← tree.toString}"
+      for (msg, i) in step.msgs.zipIdx do
+        IO.println s!"🔈[{i}] {← msg.toString}"
+  let (_, _) ← frontendM.run {} |>.run context |>.run state
+  return 0
+
+end Pantograph
+
+open Pantograph
+
+def help : IO UInt32 := do
+  IO.println "Usage: tomograph dissect FILE_NAME"
+  return 1
+
+def main (args : List String) : IO UInt32 := do
+  let command :: args := args | help
+  unsafe do
+    Pantograph.initSearch ""
+  match command with
+  | "dissect" => dissect args
+  | _ => fail s!"Unknown command {command}"

doc/rationale.md

@@ -37,21 +37,22 @@ differently from Lean in some times, but never at the sacrifice of soundness.
 - When Lean LSP says "unresolved goals", that means a proof cannot finish where
   it is supposed to finish at the end of a `by` block. Pantograph will raise the
   error in this case, since it indicates the termination of a proof search branch.
-- `pick_goal` or `swap` will not work since they run contrary to tree search
-  paradigms. However, if there are tactics which perform non-trivial operations
-  to multiple goals at the same time, this constrain could potentially be
-  relaxed at a cost of great bookkeeping overhead to the user.
 
 Pantograph cannot perform things that are inherently constrained by Lean. These
 include:
 
 - If a tactic loses track of metavariables, it will not be caught until the end
   of the proof search. This is a bug in the tactic itself.
-- Timeouts for executing tactics is not available. Maybe this will change in the
-  future.
+- Although a timeout feature exists in Pantograph, it relies on the coöperative
+  multitasking from the tactic implementation. There is nothing preventing a
+  buggy tactic from stalling Lean if it does not check for cancellation often.
+- For the same reason as above, there is no graceful way to stop a tactic which
+  leaks infinite memory. Users who wish to have this behaviour should run
+  Pantograph in a controlled environment with limited allocations. e.g.
+  Linux control groups.
 - Interceptions of parsing errors generally cannot be turned into goals (e.g.
-  `def mystery : Nat := :=`) due to Lean's parsing system.
-
+  `def mystery : Nat := :=`) due to Lean's parsing system. This question is also
+  not well-defined.
 
 ## References
 

doc/repl.md

@@ -1,5 +1,83 @@
 # REPL
 
+This documentation is about interacting with the REPL.
+
+## Examples
+
+After building the `repl`, it will be available in `.lake/build/bin/repl`.
+Execute it by either directly referring to its name, or `lake exe repl`.
+
+``` sh
+repl MODULES|LEAN_OPTIONS
+```
+
+The `repl` executable must be given with a list of modules to import. By default
+it will import nothing, not even `Init`. It can also accept lean options of the
+form `--key=value` e.g. `--pp.raw=true`.
+
+After it emits the `ready.` signal, `repl` accepts commands as single-line JSON
+inputs and outputs either an `Error:` (indicating malformed command) or a JSON
+return value indicating the result of a command execution. The command must be
+given in one of two formats
+
+```
+command { ... }
+{ "cmd": command, "payload": ... }
+```
+
+The list of available commands can be found below. An empty command aborts the
+REPL.
+
+Example: (~5k symbols)
+```
+$ repl Init
+env.catalog
+env.inspect {"name": "Nat.le_add_left"}
+```
+
+Example with `mathlib4` (~90k symbols, may stack overflow, see troubleshooting)
+
+```
+$ repl Mathlib.Analysis.Seminorm
+env.catalog
+```
+
+Example proving a theorem: (alternatively use `goal.start {"copyFrom": "Nat.add_comm"}`)
+to prime the proof
+
+```
+$ repl Init
+goal.start {"expr": "∀ (n m : Nat), n + m = m + n"}
+goal.tactic {"stateId": 0, "tactic": "intro n m"}
+goal.tactic {"stateId": 1, "tactic": "assumption"}
+goal.delete {"stateIds": [0]}
+stat {}
+goal.tactic {"stateId": 1, "tactic": "rw [Nat.add_comm]"}
+stat
+```
+where the application of `assumption` should lead to a failure.
+
+For a list of commands, see [REPL Documentation](doc/repl.md).
+
+### Project Environment
+
+To use Pantograph in a project environment, setup the `LEAN_PATH` environment
+variable so it contains the library path of lean libraries. The libraries must
+be built in advance. For example, if `mathlib4` is stored at `../lib/mathlib4`,
+the environment might be setup like this:
+
+``` sh
+LIB="../lib"
+LIB_MATHLIB="$LIB/mathlib4/.lake"
+export LEAN_PATH="$LIB_MATHLIB:$LIB_MATHLIB/aesop/build/lib:$LIB_MATHLIB/Qq/build/lib:$LIB_MATHLIB/std/build/lib"
+
+LEAN_PATH=$LEAN_PATH repl $@
+```
+The `$LEAN_PATH` executable of any project can be extracted by
+``` sh
+lake env printenv LEAN_PATH
+```
+
 ## Commands
 
 See `Pantograph/Protocol.lean` for a description of the parameters and return values in JSON.
@@ -13,29 +91,29 @@ See `Pantograph/Protocol.lean` for a description of the parameters and return va
   only the values of definitions are printed.
 * `env.save { "path": <fileName> }`, `env.load { "path": <fileName> }`: Save/Load the
   current environment to/from a file
-* `env.module_read { "module": <name }`: Reads a list of symbols from a module
+* `env.module_read { "module": <name> }`: Reads a list of symbols from a module
 * `env.describe {}`: Describes the imports and modules in the current environment
-* `options.set { key: value, ... }`: Set one or more options (not Lean options; those
-  have to be set via command line arguments.), for options, see `Pantograph/Protocol.lean`
-
-  One particular option for interest for machine learning researchers is the
-  automatic mode (flag: `"automaticMode"`).  By default it is turned on, with
-  all goals automatically resuming. This makes Pantograph act like a gym,
-  with no resumption necessary to manage your goals.
+* `options.set { key: value, ... }`: Set one or more options. These are not Lean
+  `CoreM` options; those have to be set via command line arguments.), for
+  options see below.
 * `options.print`: Display the current set of options
 * `goal.start {["name": <name>], ["expr": <expr>], ["levels": [<levels>]], ["copyFrom": <symbol>]}`:
   Start a new proof from a given expression or symbol
-* `goal.tactic {"stateId": <id>, "goalId": <id>, ...}`: Execute a tactic string on a
-  given goal. The tactic is supplied as additional key-value pairs in one of the following formats:
-  - `{ "tactic": <tactic> }`: Execute an ordinary tactic
-  - `{ "expr": <expr> }`: Assign the given proof term to the current goal
-  - `{ "have": <expr>, "binderName": <name> }`: Execute `have` and creates a branch goal
-  - `{ "calc": <expr> }`: Execute one step of a `calc` tactic. Each step must
+* `goal.tactic {"stateId": <id>, ["goalId": <id>], ["autoResume": <bool>], ...}`:
+  Execute a tactic string on a given goal site. The tactic is supplied as additional
+  key-value pairs in one of the following formats:
+  - `{ "tactic": <tactic> }`: Executes a tactic in the current mode
+  - `{ "mode": <mode> }`: Enter a different tactic mode. The permitted values
+    are `tactic` (default), `conv`, `calc`. In case of `calc`, each step must
     be of the form `lhs op rhs`. An `lhs` of `_` indicates that it should be set
     to the previous `rhs`.
-  - `{ "conv": <bool> }`: Enter or exit conversion tactic mode. In the case of
-    exit, the goal id is ignored.
-  - `{ "draft": <expr> }`: Draft an expression with `sorry`s, turning them into goals. Coupling is not allowed.
+  - `{ "expr": <expr> }`: Assign the given proof term to the current goal
+  - `{ "have": <expr>, "binderName": <name> }`: Execute `have` and creates a branch goal
+  - `{ "let": <expr>, "binderName": <name> }`: Execute `let` and creates a branch goal
+  - `{ "draft": <expr> }`: Draft an expression with `sorry`s, turning them into
+    goals. Coupling is not allowed.
+  If the `goals` field does not exist, the tactic execution has failed. Read
+  `messages` to find the reason.
 * `goal.continue {"stateId": <id>, ["branch": <id>], ["goals": <names>]}`:
   Execute continuation/resumption
   - `{ "branch": <id> }`: Continue on branch state. The current state must have no goals.
@@ -46,14 +124,25 @@ See `Pantograph/Protocol.lean` for a description of the parameters and return va
   Save/Load a goal state to/from a file. The environment is not carried with the
   state. The user is responsible to ensure the sender/receiver instances share
   the same environment.
-* `frontend.process { ["fileName": <fileName>,] ["file": <str>], invocations:
-  <bool>, sorrys: <bool>, typeErrorsAsGoals: <bool>, newConstants: <bool> }`:
+* `frontend.process { ["fileName": <fileName>,] ["file": <str>], readHeader: <bool>, inheritEnv: <bool>, invocations:
+  <string>, sorrys: <bool>, typeErrorsAsGoals: <bool>, newConstants: <bool> }`:
   Executes the Lean frontend on a file, collecting the tactic invocations
-  (`"invocations": true`), the sorrys and type errors into goal states
+  (`"invocations": output-path`), the sorrys and type errors into goal states
   (`"sorrys": true`), and new constants (`"newConstants": true`). In the case of
   `sorrys`, this command additionally outputs the position of each captured
-  `sorry`. Warning: Behaviour is unstable in case of multiple `sorry`s. Use the
-  draft tactic if possible.
+  `sorry`. Conditionally inherit the environment from executing the file.
+  Warning: Behaviour is unstable in case of multiple `sorry`s. Use the draft
+  tactic if possible.
+
+## Options
+
+The full list of options can be found in `Pantograph/Protocol.lean`. Particularly:
+- `automaticMode` (default on): Goals will not become dormant when this is
+  turned on. By default it is turned on, with all goals automatically resuming.
+  This makes Pantograph act like a gym, with no resumption necessary to manage
+  your goals.
+- `timeout` (default 0): Set `timeout` to a non-zero number to specify timeout
+  (milliseconds) for all `CoreM` and frontend operations.
 
 ## Errors
 
@@ -69,3 +158,10 @@ Common error forms:
 * `index`: Indicates an invariant maintained by the output of one command and
   input of another is broken. For example, attempting to query a symbol not
   existing in the library or indexing into a non-existent proof state.
+
+## Troubleshooting
+
+If lean encounters stack overflow problems when printing catalog, execute this before running lean:
+```sh
+ulimit -s unlimited
+```

flake.lock

@@ -5,11 +5,11 @@
         "nixpkgs-lib": "nixpkgs-lib"
       },
       "locked": {
-        "lastModified": 1730504689,
-        "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
+        "lastModified": 1751413152,
+        "narHash": "sha256-Tyw1RjYEsp5scoigs1384gIg6e0GoBVjms4aXFfRssQ=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "506278e768c2a08bec68eb62932193e341f55c90",
+        "rev": "77826244401ea9de6e3bac47c2db46005e1f30b5",
         "type": "github"
       },
       "original": {
@@ -42,11 +42,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1736388194,
-        "narHash": "sha256-ymSrd/A8Pw+9FzbxUbR7CkFHLJK1b4SnFFWg/1e0JeE=",
+        "lastModified": 1751390929,
+        "narHash": "sha256-Wl2miy8PhNF6ue3R1ssQbsVK3foJ37tYtznNq/i14YM=",
         "owner": "lenianiva",
         "repo": "lean4-nix",
-        "rev": "90f496bc0694fb97bdfa6adedfc2dc2c841a4cf2",
+        "rev": "68a07e4af5e2e56276056b172e046a2276d21aee",
         "type": "github"
       },
       "original": {
@@ -55,49 +55,35 @@
         "type": "github"
       }
     },
-    "lspec": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1728279187,
-        "narHash": "sha256-ZMqbvCqR/gHXRuIkuo7b0Yp9N1vOQR7xnrcy/SeIBoQ=",
-        "owner": "argumentcomputer",
-        "repo": "LSpec",
-        "rev": "504a8cecf8da601b9466ac727aebb6b511aae4ab",
-        "type": "github"
-      },
-      "original": {
-        "owner": "argumentcomputer",
-        "ref": "504a8cecf8da601b9466ac727aebb6b511aae4ab",
-        "repo": "LSpec",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1728500571,
-        "narHash": "sha256-dOymOQ3AfNI4Z337yEwHGohrVQb4yPODCW9MDUyAc4w=",
+        "lastModified": 1743095683,
+        "narHash": "sha256-gWd4urRoLRe8GLVC/3rYRae1h+xfQzt09xOfb0PaHSk=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "d51c28603def282a24fa034bcb007e2bcb5b5dd0",
+        "rev": "5e5402ecbcb27af32284d4a62553c019a3a49ea6",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "nixos-24.05",
+        "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "nixpkgs-lib": {
       "locked": {
-        "lastModified": 1730504152,
-        "narHash": "sha256-lXvH/vOfb4aGYyvFmZK/HlsNsr/0CVWlwYvo2rxJk3s=",
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz"
+        "lastModified": 1751159883,
+        "narHash": "sha256-urW/Ylk9FIfvXfliA1ywh75yszAbiTEVgpPeinFyVZo=",
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "rev": "14a40a1d7fb9afa4739275ac642ed7301a9ba1ab",
+        "type": "github"
       },
       "original": {
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz"
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "type": "github"
       }
     },
     "nixpkgs-lib_2": {
@@ -114,16 +100,16 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1731386116,
-        "narHash": "sha256-lKA770aUmjPHdTaJWnP3yQ9OI1TigenUqVC3wweqZuI=",
+        "lastModified": 1751943650,
+        "narHash": "sha256-7orTnNqkGGru8Je6Un6mq1T8YVVU/O5kyW4+f9C1mZQ=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "689fed12a013f56d4c4d3f612489634267d86529",
+        "rev": "88983d4b665fb491861005137ce2b11a9f89f203",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "nixos-24.05",
+        "ref": "nixos-25.05",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -132,7 +118,6 @@
       "inputs": {
         "flake-parts": "flake-parts",
         "lean4-nix": "lean4-nix",
-        "lspec": "lspec",
         "nixpkgs": "nixpkgs_2"
       }
     }

flake.nix

@@ -2,13 +2,9 @@
   description = "Pantograph";
 
   inputs = {
-    nixpkgs.url = "github:nixos/nixpkgs/nixos-24.05";
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-25.05";
     flake-parts.url = "github:hercules-ci/flake-parts";
     lean4-nix.url = "github:lenianiva/lean4-nix";
-    lspec = {
-      url = "github:argumentcomputer/LSpec?ref=504a8cecf8da601b9466ac727aebb6b511aae4ab";
-      flake = false;
-    };
   };
 
   outputs = inputs @ {
@@ -16,7 +12,6 @@
     nixpkgs,
     flake-parts,
     lean4-nix,
-    lspec,
     ...
   }:
     flake-parts.lib.mkFlake {inherit inputs;} {
@@ -37,32 +32,67 @@
           inherit system;
           overlays = [(lean4-nix.readToolchainFile ./lean-toolchain)];
         };
+        manifest = pkgs.lib.importJSON ./lake-manifest.json;
+        manifest-lspec = builtins.head manifest.packages;
         lspecLib = pkgs.lean.buildLeanPackage {
           name = "LSpec";
-          roots = ["Main" "LSpec"];
-          src = "${lspec}";
+          roots = ["LSpec"];
+          src = builtins.fetchGit {inherit (manifest-lspec) url rev;};
+        };
+        inherit (pkgs.lib.fileset) unions toSource fileFilter;
+        src = ./.;
+        set-project = unions [
+          ./Pantograph.lean
+          (fileFilter (file: file.hasExt "lean") ./Pantograph)
+        ];
+        set-test = unions [
+          (fileFilter (file: file.hasExt "lean") ./Test)
+        ];
+        src-project = toSource {
+          root = src;
+          fileset = unions [
+            set-project
+          ];
+        };
+        src-repl = toSource {
+          root = src;
+          fileset = unions [
+            set-project
+            ./Main.lean
+            ./Repl.lean
+          ];
+        };
+        src-tomograph = toSource {
+          root = src;
+          fileset = unions [
+            set-project
+            ./Tomograph.lean
+          ];
+        };
+        src-test = toSource {
+          root = src;
+          fileset = unions [
+            set-project
+            ./Repl.lean
+            set-test
+          ];
         };
         project = pkgs.lean.buildLeanPackage {
           name = "Pantograph";
           roots = ["Pantograph"];
-          src = pkgs.lib.cleanSource (pkgs.lib.cleanSourceWith {
-            src = ./.;
-            filter = path: type:
-              !(pkgs.lib.hasInfix "/Test/" path)
-              && !(pkgs.lib.hasSuffix ".md" path)
-              && !(pkgs.lib.hasSuffix "Repl.lean" path);
-          });
+          src = src-project;
         };
         repl = pkgs.lean.buildLeanPackage {
           name = "Repl";
           roots = ["Main" "Repl"];
           deps = [project];
-          src = pkgs.lib.cleanSource (pkgs.lib.cleanSourceWith {
-            src = ./.;
-            filter = path: type:
-              !(pkgs.lib.hasInfix "/Test/" path)
-              && !(pkgs.lib.hasSuffix ".md" path);
-          });
+          src = src-repl;
+        };
+        tomograph = pkgs.lean.buildLeanPackage {
+          name = "tomograph";
+          roots = ["Tomograph"];
+          deps = [project];
+          src = src-tomograph;
         };
         test = pkgs.lean.buildLeanPackage {
           name = "Test";
@@ -71,17 +101,14 @@
           # Environment`) and thats where `lakefile.lean` resides.
           roots = ["Test.Main"];
           deps = [lspecLib repl];
-          src = pkgs.lib.cleanSource (pkgs.lib.cleanSourceWith {
-            src = ./.;
-            filter = path: type:
-              !(pkgs.lib.hasInfix "Pantograph" path);
-          });
+          src = src-test;
         };
       in rec {
         packages = {
           inherit (pkgs.lean) lean lean-all;
           inherit (project) sharedLib iTree;
           inherit (repl) executable;
+          tomograph = tomograph.executable;
           default = repl.executable;
         };
         legacyPackages = {

lake-manifest.json

@@ -1,15 +1,15 @@
 {"version": "1.1.0",
  "packagesDir": ".lake/packages",
  "packages":
- [{"url": "https://github.com/lenianiva/LSpec.git",
+ [{"url": "https://github.com/argumentcomputer/LSpec.git",
    "type": "git",
    "subDir": null,
    "scope": "",
-   "rev": "c492cecd0bc473e2f9c8b94d545d02cc0056034f",
+   "rev": "a6652a48b5c67b0d8dd3930fad6390a97d127e8d",
    "name": "LSpec",
    "manifestFile": "lake-manifest.json",
-   "inputRev": "c492cecd0bc473e2f9c8b94d545d02cc0056034f",
+   "inputRev": "a6652a48b5c67b0d8dd3930fad6390a97d127e8d",
    "inherited": false,
-   "configFile": "lakefile.lean"}],
+   "configFile": "lakefile.toml"}],
  "name": "pantograph",
  "lakeDir": ".lake"}

lakefile.lean

@@ -10,6 +10,7 @@ lean_lib Pantograph {
 
 lean_lib Repl {
 }
+
 @[default_target]
 lean_exe repl {
   root := `Main
@@ -17,8 +18,14 @@ lean_exe repl {
   supportInterpreter := true
 }
 
+lean_exe tomograph {
+  root := `Tomograph
+  -- Solves the native symbol not found problem
+  supportInterpreter := true
+}
+
 require LSpec from git
-  "https://github.com/lenianiva/LSpec.git" @ "c492cecd0bc473e2f9c8b94d545d02cc0056034f"
+  "https://github.com/argumentcomputer/LSpec.git" @ "a6652a48b5c67b0d8dd3930fad6390a97d127e8d"
 lean_lib Test {
 }
 @[test_driver]

lean-toolchain

@@ -1 +1 @@
-leanprover/lean4:v4.15.0
+leanprover/lean4:v4.21.0