Source maps

package.json

@@ -13,7 +13,9 @@
     "stdlib:pack": "node scripts/pack-stdlib-to-object.js",
     "wasm:dist": "cp src/tolkfiftlib.js dist && cp src/tolkfiftlib.wasm.js dist",
     "stdlib:dist": "cp -r src/tolk-stdlib dist && cp src/stdlib.tolk.js dist",
-    "test": "yarn wasm:pack && yarn stdlib:pack && yarn jest"
+    "test": "yarn wasm:pack && yarn stdlib:pack && yarn jest",
+    "fmt": "prettier --write -l --cache .",
+    "fmt:check": "prettier --check --cache ."
   },
   "author": "TON Blockchain",
   "license": "MIT",
@@ -22,14 +24,38 @@
     "url": "git+https://github.com/ton-blockchain/tolk-js.git"
   },
   "devDependencies": {
-    "@ton/core": "^0.56.3",
+    "@ton/core": "^0.61.0",
     "@ton/crypto": "^3.3.0",
     "@types/jest": "^29.5.12",
     "jest": "^29.7.0",
     "ts-jest": "^29.2.4",
-    "typescript": "^5.5.4"
+    "typescript": "^5.5.4",
+    "prettier": "^3.6.2"
   },
   "dependencies": {
-    "arg": "^5.0.2"
+    "arg": "^5.0.2",
+    "ton-source-map": "0.2.1",
+    "ton-assembly": "0.3.1"
+  },
+  "prettier": {
+    "arrowParens": "avoid",
+    "bracketSpacing": false,
+    "printWidth": 100,
+    "semi": false,
+    "tabWidth": 2,
+    "trailingComma": "all",
+    "useTabs": false,
+    "quoteProps": "preserve",
+    "overrides": [
+      {
+        "files": [
+          "*.yaml",
+          "*.yml"
+        ],
+        "options": {
+          "tabWidth": 2
+        }
+      }
+    ]
   }
 }

src/cli.ts

@@ -13,6 +13,7 @@ async function tolkJsCli() {
     '--output-fift': String,
     '--experimental-options': String,
     '--cwd': String,
+    '--source-map': Boolean,
 
     '-v': '--version',
     '-h': '--help',
@@ -31,6 +32,7 @@ Options:
 --output-fif <filename> - output .fif file with Fift code output
 --experimental-options <names> - set experimental compiler options, comma-separated
 --cwd <path>, -C <path> — sets cwd to locate .tolk files (doesn't affect output paths)
+--source-map — collect a source map for debugging
 `)
     process.exit(0)
   }
@@ -57,6 +59,7 @@ Options:
     entrypointFileName: args._[0],
     experimentalOptions: args['--experimental-options'],
     fsReadCallback: p => fs.readFileSync(cwd ? path.join(cwd, p) : p, 'utf-8'),
+    collectSourceMap: args['--source-map'] === true,
   })
 
   if (result.status === 'error') {
@@ -71,6 +74,10 @@ Options:
       codeBoc64: result.codeBoc64,
       codeHashHex: result.codeHashHex,
       sourcesSnapshot: result.sourcesSnapshot,
+      fiftSourceMapCode: result.fiftSourceMapCode,
+      sourceMapCodeRecompiledBoc64: result.sourceMapCodeRecompiledBoc64,
+      sourceMapCodeBoc64: result.sourceMapCodeBoc64,
+      sourceMap: result.sourceMap,
     }, null, 2))
   }
 

src/index.ts

@@ -5,6 +5,8 @@ import wasmBase64 from "./tolkfiftlib.wasm.js"
 // @ts-ignore
 import stdlibContents from "./stdlib.tolk.js"
 import {realpath} from "./path-utils"
+import {Cell, runtime, text, trace} from "ton-assembly";
+import {AssemblyMapping, HighLevelMapping, SourceMap} from "ton-source-map"
 
 let wasmBinary: Uint8Array | undefined = undefined
 
@@ -19,6 +21,7 @@ export type TolkCompilerConfig = {
   withStackComments?: boolean
   withSrcLineComments?: boolean
   experimentalOptions?: string
+  collectSourceMap?: boolean
 }
 
 export type TolkResultSuccess = {
@@ -27,9 +30,24 @@ export type TolkResultSuccess = {
   codeBoc64: string
   codeHashHex: string
   stderr: string
+  fiftSourceMapCode?: string
+  sourceMapCodeRecompiledBoc64?: string
+  sourceMapCodeBoc64?: string
+  sourceMap?: SourceMap
   sourcesSnapshot: { filename: string, contents: string }[]
 }
 
+type TolkCompilerResultSuccess = {
+  status: "ok"
+  fiftCode: string
+  codeBoc64: string
+  codeHashHex: string
+  stderr: string
+  fiftSourceMapCode?: string
+  sourceMapCodeBoc64?: string
+  sourceMap?: HighLevelMapping
+}
+
 export type TolkResultError = {
   status: "error"
   message: string
@@ -122,17 +140,83 @@ export async function runTolkCompiler(compilerConfig: TolkCompilerConfig): Promi
     withStackComments: compilerConfig.withStackComments,
     withSrcLineComments: compilerConfig.withSrcLineComments,
     experimentalOptions: compilerConfig.experimentalOptions,
+    collectSourceMap: compilerConfig.collectSourceMap,
   })
 
   const configStrPtr = copyToCStringAllocating(mod, configStr)
   allocatedPointers.push(configStrPtr)
 
   const resultPtr = mod._tolk_compile(configStrPtr, callbackPtr)
   allocatedPointers.push(resultPtr)
-  const result: TolkResultSuccess | TolkResultError = JSON.parse(copyFromCString(mod, resultPtr))
+  const result: TolkCompilerResultSuccess | TolkResultError = JSON.parse(copyFromCString(mod, resultPtr))
 
   allocatedPointers.forEach(ptr => mod._free(ptr))
   mod.removeFunction(callbackPtr)
 
-  return result.status === 'error' ? result : {...result, sourcesSnapshot}
+  if (result.status === 'error') {
+    return result
+  }
+
+  if (compilerConfig.collectSourceMap) {
+    // When we compile with a source map enabled, the compiler generates special DEBUGMARK %id
+    // instructions that describe the start of a code section with a specific ID.
+    // These instructions, along with the rest of the Fifth code, are compiled into "poisoned"
+    // bitcode.
+    // The result of this compilation is stored in the `sourceMapCodeBoc64` field.
+    //
+    // The code generated in this way is not runnable, since the DEBUGMARK instruction is
+    // unknown to TVM, running such code directly will cause TVM to crash.
+    //
+    // And this is where the further code comes into play.
+    //
+    // Its task is to disassemble bitcode back into instructions, including DEBUGMARK, and
+    // compile it back into bitcode.
+    // Thanks to DEBUGMARK instructions, upon recompilation, TASM can map of each instruction
+    // and the debug section, thus getting a complete source code map that is accurately down
+    // to the specific TVM instruction.
+
+    const sourceMapCodeCell = Cell.fromBase64(result.sourceMapCodeBoc64 ?? result.codeBoc64)
+    const [cleanCell, mapping] = recompileCell(sourceMapCodeCell);
+    const assemblyMapping: AssemblyMapping = trace.createMappingInfo(mapping)
+
+    if (result.sourceMap === undefined) {
+      console.warn('Source map was not generated. This is probably a bug in Tolk compiler.')
+    }
+
+    return {
+      ...result,
+      codeBoc64: result.codeBoc64,
+      sourceMapCodeRecompiledBoc64: cleanCell.toBoc().toString('base64'),
+      sourceMapCodeBoc64: result.sourceMapCodeBoc64,
+      sourceMap: {
+        highlevelMapping: result.sourceMap ?? emptyHighlevelMapping,
+        assemblyMapping,
+        recompiledCode: cleanCell.toBoc().toString('base64'),
+      },
+      sourcesSnapshot,
+    }
+  }
+
+  return {...result, sourcesSnapshot, sourceMap: undefined}
+}
+
+function recompileCell(cell: Cell): [Cell, runtime.Mapping] {
+  const instructions = runtime.decompileCell(cell);
+  const assembly = text.print(instructions);
+
+  const parseResult = text.parse("out.tasm", assembly);
+  if (parseResult.$ === "ParseFailure") {
+    throw new Error("Cannot parse resulting text Assembly");
+  }
+
+  return runtime.compileCellWithMapping(parseResult.instructions, {skipRefs: true});
 }
+
+const emptyHighlevelMapping: HighLevelMapping = {
+  version: "0",
+  language: "tolk",
+  compiler_version: "",
+  files: [],
+  globals: [],
+  locations: [],
+};