diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4be085e..ae3fc6a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -26,7 +26,7 @@ jobs:
         run: |
           sudo apt install libx11-dev libxtst-dev libxkbcommon-dev libdbus-1-dev libwxgtk3.0-gtk3-dev
       - name: Run test suite
-        run: cargo make test
+        run: cargo make test-binary
       - name: Build
         run: |
           cargo make build-binary
@@ -41,6 +41,8 @@ jobs:
       - name: Install cargo-make
         run: |
           cargo install --force cargo-make
+      - name: Run test suite
+        run: cargo make test-binary --env NO_X11=true
       - name: Build
         run: cargo make build-binary --env NO_X11=true
   
@@ -53,6 +55,8 @@ jobs:
       - name: Install cargo-make
         run: |
           cargo install --force cargo-make
+      - name: Run test suite
+        run: cargo make test-binary --env BUILD_ARCH=aarch64-apple-darwin
       - name: Build
         run: |
           cargo make build-macos-arm-binary
diff --git a/Makefile.toml b/Makefile.toml
index 101d8d8..2cd02ba 100644
--- a/Makefile.toml
+++ b/Makefile.toml
@@ -26,6 +26,10 @@ command = "${EXEC_PATH}"
 args = ["${@}"]
 dependencies = ["build-binary"]
 
+[tasks.test-binary]
+script_runner = "@rust"
+script = { file = "scripts/test_binary.rs" }
+
 # Windows
 
 [tasks.build-windows-resources]
@@ -96,10 +100,6 @@ dependencies=["create-app-image"]
 
 # Test runs
 
-[tasks.test]
-command = "cargo"
-args = ["test", "--workspace", "--exclude", "espanso-modulo", "--exclude", "espanso-ipc", "--no-default-features"]
-
 [tasks.test-output]
 command = "cargo"
 args = ["test", "--workspace", "--exclude", "espanso-modulo", "--exclude", "espanso-ipc", "--no-default-features", "--", "--nocapture"]
\ No newline at end of file
diff --git a/scripts/test_binary.rs b/scripts/test_binary.rs
new file mode 100644
index 0000000..31de66f
--- /dev/null
+++ b/scripts/test_binary.rs
@@ -0,0 +1,78 @@
+//! ```cargo
+//! [dependencies]
+//! envmnt = "*"
+//! ```
+
+use std::process::Command;
+
+#[derive(Debug, PartialEq)]
+enum Profile {
+  Debug,
+  Release,
+}
+
+fn main() {
+  let profile = if envmnt::get_or_panic("RELEASE") == "true" {
+    Profile::Release
+  } else {
+    Profile::Debug
+  };
+
+  println!("Using profile: {:?}", profile);
+
+  let wayland = envmnt::get_or("NO_X11", "false") == "true";
+  if wayland {
+    println!("Using Wayland feature");
+  }
+
+  let mut args = Vec::new();
+
+  args.push("test");
+  args.push("--workspace");
+  args.push("--exclude");
+  args.push("espanso-modulo");
+  args.push("--exclude");
+  args.push("espanso-ipc");
+
+  if profile == Profile::Release {
+    args.push("--release");
+  }
+
+  let override_target_arch = envmnt::get_or("BUILD_ARCH", "current");
+  if override_target_arch != "current" {
+    args.push("--target");
+    args.push(&override_target_arch);
+  }
+
+  let mut features = Vec::new();
+  if wayland {
+    features.push("wayland");
+  }
+
+  let features_flag = features.join(" ");
+
+  args.push("--no-default-features");
+  args.push("--features");
+  args.push(&features_flag);
+
+  println!("Calling with args: {:?}", args);
+
+  let mut cmd = Command::new("cargo");
+  cmd.args(&args);
+
+  // Remove cargo/rust-specific env variables, as otherwise they mess up the
+  // nested cargo build call.
+  let all_vars = envmnt::vars();
+  for (key, _) in all_vars {
+    if key.starts_with("CARGO") || key.starts_with("RUST") {
+      //println!("Removing {}", key);
+      cmd.env_remove(key);
+    }
+  }
+
+  let mut handle = cmd.spawn().expect("cargo test failed");
+  let result = handle.wait().expect("unable to read cargo exit status");
+  if !result.success() {
+    panic!("cargo test failed");
+  }
+}