Example for custom WASM builds with Rust (#221)

Add documentation on how to create custom WASM modules for use with this package. An example shows how to include Rust extensions in WASM builds.

Example for custom WASM builds with Rust (#221)
Add documentation on how to create custom WASM modules for use with this package. An example shows how to include Rust extensions in WASM builds.
5743cd03 · Simon Binder · GitHub · 2ffd8716 · 5743cd03 · 5743cd03
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
+{
+    "rust-analyzer.linkedProjects": [
+        "sqlite3/example/custom_wasm_build/Cargo.toml"
+    ]
+}
--- a/sqlite3/README.md
+++ b/sqlite3/README.md
@@ -203,3 +203,19 @@ cmake --build .dart_tool/sqlite3_build/ -t output -j
 The `output` target copies `sqlite3.wasm` and `sqlite3.debug.wasm` to `example/web`.

 (Of course, you can also run the build in any other directory than `.dart_tool/sqite3_build` if you want to).
+
+### Customizing the WASM module
+
+The build scripts in this repository, which are also used for the default distribution of `sqlite3.wasm`
+attached to releases, are designed to mirror the options used by `sqlite3_flutter_libs`.
+If you want to use different options, or include custom extensions in the WASM module, you can customize
+the build setup.
+
+To use regular sqlite3 sources with different compile-time options, alter `assets/wasm/sqlite_cfg.h` and
+re-run the build as described in [compiling](#compiling).
+Including additional extensions written in C is possible by adapting the `CMakeLists.txt` in
+`assets/wasm`.
+
+A simple example demonstrating how to include Rust-based extensions is included in `example/custom_wasm_build`.
+The readme in that directory explains the build process in detail, but you still need the WASI/Clang toolchains
+described in the [setup section](#linux).
--- a/sqlite3/example/custom_wasm_build/.cargo/config.toml
+++ b/sqlite3/example/custom_wasm_build/.cargo/config.toml
+[build]
+rustflags = ["-C", "linker-plugin-lto"]
--- a/sqlite3/example/custom_wasm_build/.gitignore
+++ b/sqlite3/example/custom_wasm_build/.gitignore
+target/
+Cargo.lock
--- a/sqlite3/example/custom_wasm_build/Cargo.toml
+++ b/sqlite3/example/custom_wasm_build/Cargo.toml
+[package]
+name = "custom_wasm_build"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["staticlib"]
+
+[dependencies]
+
+[build-dependencies]
+cc = "1.0.46"
+cmake = "0.1.50"
+
+[profile.release]
+opt-level = "z"
+lto=true
--- a/sqlite3/example/custom_wasm_build/README.md
+++ b/sqlite3/example/custom_wasm_build/README.md
+## custom wasm modules
+
+This example demonstrates how to build a custom `sqlite3.wasm` module than can be
+used with `package:sqlite3`.
+Using custom wasm modules is useful to include additional extensions in sqlite3, or to
+use different compile-time options.
+
+This example uses existing C source files also used by the default configuration:
+
+- `sqlite3.c` for the actual SQLite library.
+- [`helpers.c`](https://github.com/simolus3/sqlite3.dart/blob/main/sqlite3/assets/wasm/helpers.c) from this repository, which defines a VFS wrapper for virtual filesystem implementations provided from Dart.
+- Our `os_web.c` file is _not_ included. It contains the implementation for
+  `sqlite3_os_init` and `sqlite3_os_end`, which are implemented in this custom
+  extension instead. `sqlite3_os_init` is a suitable hook to load your sqlite3
+  extensions.
+
+In this example, the extension is a simple Rust library printing a hello message
+to the web console when the module is loaded.
+
+## Setup
+
+We're currently using the libc from [WASI](https://wasi.dev/) to compile sqlite3,
+so the easiest approach is to compile your custom extensions with that as well:
+
+```
+rustup target add wasm32-wasi
+```
+
+Additionally, you need to download WASI compiler builtins and the associated libc
+as described in the [build instructions](https://github.com/simolus3/sqlite3.dart/tree/rust-wasm-build/sqlite3#compiling).
+
+## Building
+
+The `build.rs` file from this example is responsible for compiling sqlite3 to
+WebAssembly object files. We're not using a static library because that seems to
+break the `--export-dynamic` option used in the final linking step to expose the
+relevant functions.
+
+To download and compile sqlite3 as well as the patched `sqlite3_os_init` function
+for this example, run
+
+```
+WASI_SYSROOT=/path/to/wasi/ CC=/path/to/clang cargo build --target wasm32-wasi
+```
+
+Or, to compile a release build optimized for size:
+
+```
+WASI_SYSROOT=/path/to/wasi/ CC=/path/to/clang cargo build --target wasm32-wasi --release
+```
+
+Cargo compiles sqlite3 to WASM object files, the Rust part is compiled into a static
+library.
+A Dart script (`link.dart`) can be used to link them together into a `sqlite3.wasm`
+file loadable by the `sqlite3` package:
+
+```
+CC=/path/to/clang dart run link.dart target/wasm32-wasi/debug
+CC=/path/to/clang dart run link.dart target/wasm32-wasi/release
+```
+
+As an additional transformation step, running `wasm-opt -O4` on the resulting
+WASM file may optimize it further.
--- a/sqlite3/example/custom_wasm_build/build.rs
+++ b/sqlite3/example/custom_wasm_build/build.rs
+use std::{env, path::PathBuf};
+
+use cmake::Config;
+
+fn main() {
+    let sysroot =
+        env::var("WASI_SYSROOT").unwrap_or_else(|_| "/usr/share/wasi-sysroot".to_string());
+
+    let cmake_dir = Config::new("../../assets/wasm/")
+        .configure_arg("--toolchain")
+        .configure_arg(std::fs::canonicalize("../../assets/wasm/toolchain.cmake").unwrap())
+        .define("wasi_sysroot", &sysroot)
+        .build_target("sqlite3_opt_lib")
+        .build_target("help") // We only need the sources
+        .build();
+    let sqlite3_src = cmake_dir.join("build/_deps/sqlite3-src/");
+
+    let mut c_build = cc::Build::new();
+    let objects = c_build
+        .target("wasm32-unknown-wasi")
+        .cargo_warnings(false)
+        .flag("--sysroot")
+        .flag(&sysroot)
+        .file(sqlite3_src.join("sqlite3.c"))
+        .file("../../assets/wasm/helpers.c")
+        .flag("-flto=thin")
+        .include(&sqlite3_src)
+        .include("../../assets/wasm/")
+        .define("_HAVE_SQLITE_CONFIG_H", None)
+        .define("SQLITE_API", "__attribute__((visibility(\"default\")))")
+        // Ideally we should be able to compile this into a static library and use that one, but
+        // for some reasons that drops all exported symbols. So we're compiling to objects and
+        // we only compile Rust code to a static library. Then we use the clang driver to link
+        // these objects and the static Rust library in one go.
+        .compile_intermediates();
+
+    let output_dir = get_output_path();
+    for object in objects {
+        // The file name is something like <hash>-sqlite3.o
+        let file_name = object.file_name().unwrap().to_str().unwrap().to_owned();
+        let (_, file_name) = file_name.split_once("-").unwrap();
+
+        std::fs::copy(object, output_dir.join(file_name)).unwrap();
+    }
+}
+
+fn get_output_path() -> PathBuf {
+    let mut out = PathBuf::from(env::var("OUT_DIR").unwrap());
+
+    loop {
+        match out.file_name() {
+            Some(name) if name == "build" => {
+                break;
+            }
+            _ => out = out.parent().unwrap().to_path_buf(),
+        }
+    }
+
+    out.parent().unwrap().to_path_buf()
+}
--- a/sqlite3/example/custom_wasm_build/link.dart
+++ b/sqlite3/example/custom_wasm_build/link.dart
+import 'dart:io';
+
+void main(List<String> args) {
+  if (args.length != 1) {
+    print("Usage: dart link.dart target/wasm32-wasi/<buildtype>");
+  }
+
+  final directory = Directory(args[0]);
+  final entries = <String>[];
+
+  for (final entry in directory.listSync()) {
+    if (entry.path.endsWith('.o') || entry.path.endsWith('.a')) {
+      entries.add(entry.path);
+    }
+  }
+
+  final clang = Platform.environment['CC'] ?? 'clang';
+  final sysroot =
+      Platform.environment['WASI_SYSROOT'] ?? '/usr/share/wasi-sysroot';
+
+  final process = Process.runSync(clang, [
+    '--target=wasm32-unknown-wasi',
+    '--sysroot=$sysroot',
+    '-flto',
+    ...entries,
+    '-o',
+    'sqlite3.wasm',
+    '-nostartfiles',
+    '-Wl,--no-entry',
+    '-Wl,--export-dynamic',
+    '-Wl,--import-memory',
+    '-v',
+  ]);
+
+  if (process.exitCode != 0) {
+    print(
+        'Could not link: ${process.exitCode}, ${process.stderr}, ${process.stdout}');
+  }
+}
--- a/sqlite3/example/custom_wasm_build/src/lib.rs
+++ b/sqlite3/example/custom_wasm_build/src/lib.rs
+use std::ffi::c_int;
+
+#[no_mangle]
+pub extern "C" fn sqlite3_os_init() -> c_int {
+    // This would be a good place to set up extensions.
+    unsafe {
+        // package:sqlite3 provides access to Dart's print function via dart.error_log
+        wasm::dartLogError("Hello from a custom Rust build!".as_ptr().cast());
+    }
+
+    return 0;
+}
+
+#[no_mangle]
+pub extern "C" fn sqlite3_os_end() -> c_int {
+    return 0;
+}
+
+mod wasm {
+    use std::ffi::c_char;
+
+    #[link(wasm_import_module = "dart")]
+    extern "C" {
+        #[link_name = "error_log"]
+        pub fn dartLogError(msg: *const c_char);
+    }
+}