feat(migrate): add test cases and rendering implementation

Cargo.lock

@@ -509,6 +509,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "dunce",
+ "fs_extra",
  "glob",
  "include_dir",
  "lazy_static",
@@ -616,6 +617,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "fs_extra"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
+
 [[package]]
 name = "fuchsia-cprng"
 version = "0.1.1"

espanso-migrate/Cargo.toml

@@ -15,9 +15,10 @@ dunce = "1.0.1"
 walkdir = "2.3.1"
 yaml-rust = "0.4.5"
 path-slash = "0.1.4"
+tempdir = "0.3.7"
+fs_extra = "1.2.0"
 
 [dev-dependencies]
-tempdir = "0.3.7"
 tempfile = "3.2.0"
 include_dir = { version = "0.6.0", features = ["search"] }
 test-case = "1.1.0"

espanso-migrate/src/convert.rs

@@ -18,9 +18,14 @@
  */
 
 use std::{cmp::Ordering, collections::HashMap, path::PathBuf};
-use yaml_rust::{yaml::Hash, Yaml, YamlEmitter};
+use yaml_rust::{yaml::Hash, Yaml};
 
-pub fn convert(input_files: HashMap<String, Hash>) -> HashMap<String, Hash> {
+pub struct ConvertedFile {
+  pub origin: String,
+  pub content: Hash,
+}
+
+pub fn convert(input_files: HashMap<String, Hash>) -> HashMap<String, ConvertedFile> {
   let mut output_files = HashMap::new();
 
   let sorted_input_files = sort_input_files(&input_files);
@@ -57,10 +62,13 @@ pub fn convert(input_files: HashMap<String, Hash>) -> HashMap<String, Hash> {
 
       let output_yaml = output_files
         .entry(match_output_path.clone())
-        .or_insert(Hash::new());
+        .or_insert(ConvertedFile { 
+          origin: input_path.to_string(),
+          content: Hash::new(),
+        });
 
       if let Some(global_vars) = yaml_global_vars {
-        let output_global_vars = output_yaml
+        let output_global_vars = output_yaml.content
           .entry(Yaml::String("global_vars".to_string()))
           .or_insert(Yaml::Array(Vec::new()));
         if let Yaml::Array(out_global_vars) = output_global_vars {
@@ -71,7 +79,7 @@ pub fn convert(input_files: HashMap<String, Hash>) -> HashMap<String, Hash> {
       }
 
       if let Some(matches) = yaml_matches {
-        let output_matches = output_yaml
+        let output_matches = output_yaml.content
           .entry(Yaml::String("matches".to_string()))
           .or_insert(Yaml::Array(Vec::new()));
         if let Yaml::Array(out_matches) = output_matches {
@@ -162,7 +170,10 @@ pub fn convert(input_files: HashMap<String, Hash>) -> HashMap<String, Hash> {
         output_yaml.insert(Yaml::String(key_name.to_string()), Yaml::Array(includes));
       }
 
-      output_files.insert(config_output_path, output_yaml);
+      output_files.insert(config_output_path, ConvertedFile {
+        origin: input_path,
+        content: output_yaml,
+      });
     }
 
     // TODO: create config file

espanso-migrate/src/lib.rs

@@ -28,11 +28,16 @@ extern crate include_dir;
 #[cfg(test)]
 extern crate test_case;
 
+use std::path::Path;
+
 use anyhow::Result;
+use fs_extra::dir::CopyOptions;
+use tempdir::TempDir;
 use thiserror::Error;
 
 mod convert;
 mod load;
+mod render;
 
 // TODO: implement
 // Use yaml-rust with "preserve-order" = true
@@ -51,33 +56,94 @@ mod load;
 // creates the new config on a new temporary directory and then "swaps"
 // the old with the new one
 
-// TODO: test case with packages
-
-// TODO: keep other non-standard directories such as "images/" and "script/"
-
 // TODO: test also with non-lowercase file names
 
-// TODO: test packages in another directory (a possible strategy is to copy
-// the packages dir and the config dir into a temporary one, with the packages
-// as a directory at the same level of user/)
+pub fn migrate(config_dir: &Path, packages_dir: &Path, output_dir: &Path) -> Result<()> {
+  if !config_dir.is_dir() {
+    return Err(MigrationError::InvalidConfigDir.into());
+  }
 
-// TODO: when dumping the output file, remove the front-matter at the top (generated by YamlEmitter)
-// and insert a comment with "Automatically generated from {{file_name}} by migration tool"
+  let working_dir = TempDir::new("espanso-migration")?;
+
+  fs_extra::dir::copy(
+    config_dir,
+    working_dir.path(),
+    &CopyOptions {
+      content_only: true,
+      ..Default::default()
+    },
+  )?;
+
+  // If packages are located within the config_dir, no need to copy them in
+  // the working directory
+  if packages_dir.parent() != Some(config_dir) {
+    fs_extra::dir::copy(packages_dir, working_dir.path(), &CopyOptions::new())?;
+  }
+
+  // Create the output directory
+  if output_dir.exists() {
+    return Err(MigrationError::OutputDirAlreadyPresent.into());
+  }
+
+  std::fs::create_dir_all(output_dir)?;
+
+  // Convert the configurations
+  let legacy_files = load::load(working_dir.path())?;
+  let converted_files = convert::convert(legacy_files);
+  let rendered_files = render::render(converted_files)?;
+
+  for (file, content) in rendered_files {
+    let target = output_dir.join(file);
+
+    if let Some(parent) = target.parent() {
+      if !parent.is_dir() {
+        std::fs::create_dir_all(parent)?;
+      }
+    }
+
+    std::fs::write(target, content)?;
+  }
+
+  // Copy all non-YAML directories
+  for entry in std::fs::read_dir(working_dir.path())? {
+    let entry = entry?;
+    let path = entry.path();
+    if path.is_dir() {
+      let dir_name = path.file_name();
+      if let Some(name) = dir_name.map(|s| s.to_string_lossy().to_string().to_lowercase()) {
+        if name != "user" && name != "packages" {
+          fs_extra::dir::copy(path, output_dir, &CopyOptions::new())?;
+        }
+      }
+    }
+  }
+
+  Ok(())
+}
+
+#[derive(Error, Debug)]
+pub enum MigrationError {
+  #[error("invalid config directory")]
+  InvalidConfigDir,
+
+  #[error("output directory already present")]
+  OutputDirAlreadyPresent,
+}
 
 #[cfg(test)]
 mod tests {
-  use std::{collections::HashMap, fs::create_dir_all, path::{Path}};
+  use std::{collections::HashMap, fs::create_dir_all, path::Path};
 
   use super::*;
   use include_dir::{include_dir, Dir};
   use tempdir::TempDir;
   use test_case::test_case;
 
-  use pretty_assertions::{assert_eq as assert_peq};
-use yaml_rust::{yaml::Hash, Yaml};
+  use pretty_assertions::assert_eq as assert_peq;
+  use yaml_rust::{yaml::Hash, Yaml};
 
   fn run_with_temp_dir(test_data: &Dir, action: impl FnOnce(&Path, &Path)) {
-    let tmp_dir = TempDir::new("espanso-migration").unwrap();
+    let tmp_dir = TempDir::new("espanso-migrate").unwrap();
     let tmp_path = tmp_dir.path();
     let legacy_path = tmp_dir.path().join("legacy");
     let expected_path = tmp_dir.path().join("expected");
@@ -88,7 +154,7 @@ use yaml_rust::{yaml::Hash, Yaml};
       let entry_path_str = entry_path.to_string_lossy().to_string();
       if entry_path_str.is_empty() {
         continue;
-      } 
+      }
 
       let target = tmp_path.join(entry_path);
 
@@ -96,7 +162,7 @@ use yaml_rust::{yaml::Hash, Yaml};
         create_dir_all(target).unwrap();
       } else {
         std::fs::write(target, test_data.get_file(entry_path).unwrap().contents()).unwrap();
-      }      
+      }
     }
 
     action(&legacy_path, &expected_path);
@@ -109,30 +175,54 @@ use yaml_rust::{yaml::Hash, Yaml};
   }
 
   fn to_sorted_hash(hash: &Hash) -> Vec<(String, &Yaml)> {
-    let mut tuples: Vec<(String, &Yaml)> = hash.into_iter().map(|(k, v)| (k.as_str().unwrap().to_string(), v)).collect();
+    let mut tuples: Vec<(String, &Yaml)> = hash
+      .into_iter()
+      .map(|(k, v)| (k.as_str().unwrap().to_string(), v))
+      .collect();
     tuples.sort_by_key(|(k, _)| k.clone());
     tuples
   }
 
+  fn list_files_in_dir(dir: &Path) -> Vec<String> {
+    let prefix = dir.to_string_lossy().to_string();
+    fs_extra::dir::get_dir_content(&dir)
+      .unwrap()
+      .files
+      .into_iter()
+      .map(|file| file.trim_start_matches(&prefix).to_string())
+      .collect()
+  }
+
   static SIMPLE_CASE: Dir = include_dir!("test/simple");
   static BASE_CASE: Dir = include_dir!("test/base");
   static ALL_PARAMS_CASE: Dir = include_dir!("test/all_params");
+  static OTHER_DIRS_CASE: Dir = include_dir!("test/other_dirs");
 
   #[test_case(&SIMPLE_CASE; "simple case")]
   #[test_case(&BASE_CASE; "base case")]
   #[test_case(&ALL_PARAMS_CASE; "all config parameters case")]
+  #[test_case(&OTHER_DIRS_CASE; "other directories case")]
   fn test_migration(test_data: &Dir) {
     run_with_temp_dir(test_data, |legacy, expected| {
-      let legacy_files = load::load(legacy).unwrap();
+      let tmp_out_dir = TempDir::new("espanso-migrate-out").unwrap();
+      let output_dir = tmp_out_dir.path().join("out");
+
+      migrate(legacy, &legacy.join("packages"), &output_dir).unwrap();
+
+      let converted_files = load::load(&output_dir).unwrap();
+
+      // Verify configuration content
       let expected_files = load::load(expected).unwrap();
-
-      let converted_files = convert::convert(legacy_files);
-
       assert_eq!(converted_files.len(), expected_files.len());
-
-      for (file, content) in to_sorted_list(converted_files) {
-        assert_peq!(to_sorted_hash(&content), to_sorted_hash(&expected_files.get(&file).unwrap()));
+      for (file, converted) in to_sorted_list(converted_files) {
+        assert_peq!(
+          to_sorted_hash(&converted),
+          to_sorted_hash(&expected_files.get(&file).unwrap())
+        );
       }
+
+      // Verify file structure
+      assert_peq!(list_files_in_dir(expected), list_files_in_dir(&output_dir));
     });
   }
 }

espanso-migrate/src/render.rs

@@ -0,0 +1,49 @@
+/*
+ * This file is part of espanso.
+ *
+ * Copyright (C) 2019-2021 Federico Terzi
+ *
+ * espanso is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * espanso is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with espanso.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+use anyhow::Result;
+use std::collections::HashMap;
+use yaml_rust::{Yaml, YamlEmitter};
+
+use crate::convert::ConvertedFile;
+
+pub fn render(converted_files: HashMap<String, ConvertedFile>) -> Result<Vec<(String, String)>> {
+  let mut output = Vec::new();
+  for (file, content) in converted_files {
+    let rendered = render_file(content)?;
+    output.push((file, rendered));
+  }
+  Ok(output)
+}
+
+fn render_file(file: ConvertedFile) -> Result<String> {
+  let mut dump_str = String::new();
+  let mut emitter = YamlEmitter::new(&mut dump_str);
+  emitter.dump(&Yaml::Hash(file.content))?;
+
+  let lines: Vec<&str> = dump_str.lines().collect();
+  let header = format!("# Original file: {}", file.origin);
+  let mut output_lines: Vec<&str> = vec!["# Automatically generated by espanso migration tool", &header, ""];
+  if !lines.is_empty() {
+    output_lines.extend(&lines[1..]);
+  }
+
+  let output = output_lines.join("\n");
+  Ok(output)
+}

espanso-migrate/test/other_dirs/expected/config/default.yml

@@ -0,0 +1 @@
+backend: Clipboard

espanso-migrate/test/other_dirs/expected/config/disabled.yml

@@ -0,0 +1,3 @@
+filter_title: "Disabled program"
+
+enable: false

espanso-migrate/test/other_dirs/expected/match/base.yml

@@ -0,0 +1,12 @@
+global_vars:
+  - name: "name"
+    type: "dummy"
+    params:
+      echo: "John"
+
+matches:
+  - name: ":hi"
+    trigger: "Hello"
+  
+  - name: ":greet"
+    trigger: "Hi {{name}}"

espanso-migrate/test/other_dirs/expected/scripts/test.py

@@ -0,0 +1 @@
+print("this is a test")

espanso-migrate/test/other_dirs/legacy/default.yml

@@ -0,0 +1,14 @@
+backend: Clipboard
+
+global_vars:
+  - name: "name"
+    type: "dummy"
+    params:
+      echo: "John"
+
+matches:
+  - name: ":hi"
+    trigger: "Hello"
+  
+  - name: ":greet"
+    trigger: "Hi {{name}}"

espanso-migrate/test/other_dirs/legacy/scripts/test.py

@@ -0,0 +1 @@
+print("this is a test")

espanso-migrate/test/other_dirs/legacy/user/disabled.yml

@@ -0,0 +1,3 @@
+filter_title: "Disabled program"
+
+enable_active: false