Continue the work on the new config module

Cargo.lock

@@ -232,7 +232,9 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "glob",
+ "lazy_static",
  "log",
+ "regex",
  "serde",
  "serde_yaml",
  "tempdir",

espanso-config/Cargo.toml

@@ -11,6 +11,9 @@ thiserror = "1.0.23"
 serde = { version = "1.0.123", features = ["derive"] }
 serde_yaml = "0.8.17"
 glob = "0.3.0"
+regex = "1.4.3"
+lazy_static = "1.4.0"
+
 
 [dev-dependencies]
 tempdir = "0.3.7"

espanso-config/src/config/mod.rs

@@ -9,7 +9,7 @@ mod macro_util;
 pub struct Config {
   pub label: Option<String>,
   //pub backend: 
-  pub match_files: Vec<String>,
+  pub match_paths: HashSet<String>,
 }
 
 impl Config {

espanso-config/src/config/path.rs

@@ -1,12 +1,24 @@
-use std::collections::HashSet;
+use std::{collections::HashSet, path::{Path, PathBuf}};
 
 use glob::glob;
 use log::error;
+use regex::Regex;
 
-pub fn calculate_paths<'a>(glob_patterns: impl Iterator<Item = &'a String>) -> HashSet<String> {
+lazy_static! {
+  static ref ABSOLUTE_PATH: Regex = Regex::new(r"(?m)^([a-zA-Z]:/|/).*$").unwrap();
+}
+
+pub fn calculate_paths<'a>(base_dir: &Path, glob_patterns: impl Iterator<Item = &'a String>) -> HashSet<String> {
   let mut path_set = HashSet::new();
   for glob_pattern in glob_patterns {
-    let entries = glob(glob_pattern);
+    // Handle relative and absolute paths appropriately
+    let pattern = if ABSOLUTE_PATH.is_match(glob_pattern) {
+      glob_pattern.clone()
+    } else {
+      format!("{}/{}", base_dir.to_string_lossy(), glob_pattern)
+    };
+
+    let entries = glob(&pattern);
     match entries {
       Ok(paths) => {
         for path in paths {
@@ -34,18 +46,26 @@ pub fn calculate_paths<'a>(glob_patterns: impl Iterator<Item = &'a String>) -> H
 }
 
 #[cfg(test)]
-mod tests {
+pub mod tests {
-  use std::fs::create_dir_all;
+  use std::{fs::create_dir_all, path::Path};
 
   use super::*;
   use tempdir::TempDir;
 
-  #[test]
+  pub fn use_test_directory(callback: impl FnOnce(&Path, &Path, &Path)) {
-  fn calculate_paths_works_correctly() {
     let dir = TempDir::new("tempconfig").unwrap();
     let match_dir = dir.path().join("match");
     create_dir_all(&match_dir).unwrap();
 
+    let config_dir = dir.path().join("config");
+    create_dir_all(&config_dir).unwrap();
+
+    callback(&dir.path(), &match_dir, &config_dir);
+  }
+
+  #[test]
+  fn calculate_paths_relative_paths() {
+    use_test_directory(|base, match_dir, config_dir| {
       let sub_dir = match_dir.join("sub");
       create_dir_all(&sub_dir).unwrap();
 
@@ -54,19 +74,48 @@ mod tests {
       std::fs::write(match_dir.join("_sub.yml"), "test").unwrap();
       std::fs::write(sub_dir.join("sub.yml"), "test").unwrap();
 
-    let result = calculate_paths(vec![
+      let result = calculate_paths(base, vec![
-      format!("{}/**/*.yml", dir.path().to_string_lossy()),
+        "**/*.yml".to_string(),
-      format!("{}/match/sub/*.yml", dir.path().to_string_lossy()),
+        "match/sub/*.yml".to_string(),
         // Invalid path
         "invalid".to_string(),
       ].iter());
 
       let mut expected = HashSet::new();
-    expected.insert(format!("{}/match/base.yml", dir.path().to_string_lossy()));
+      expected.insert(format!("{}/match/base.yml", base.to_string_lossy()));
-    expected.insert(format!("{}/match/another.yml", dir.path().to_string_lossy()));
+      expected.insert(format!("{}/match/another.yml", base.to_string_lossy()));
-    expected.insert(format!("{}/match/_sub.yml", dir.path().to_string_lossy()));
+      expected.insert(format!("{}/match/_sub.yml", base.to_string_lossy()));
-    expected.insert(format!("{}/match/sub/sub.yml", dir.path().to_string_lossy()));
+      expected.insert(format!("{}/match/sub/sub.yml", base.to_string_lossy()));
 
       assert_eq!(result, expected);
+    });
+  }
+
+  #[test]
+  fn calculate_paths_absolute_paths() {
+    use_test_directory(|base, match_dir, config_dir| {
+      let sub_dir = match_dir.join("sub");
+      create_dir_all(&sub_dir).unwrap();
+
+      std::fs::write(match_dir.join("base.yml"), "test").unwrap();
+      std::fs::write(match_dir.join("another.yml"), "test").unwrap();
+      std::fs::write(match_dir.join("_sub.yml"), "test").unwrap();
+      std::fs::write(sub_dir.join("sub.yml"), "test").unwrap();
+
+      let result = calculate_paths(base, vec![
+        format!("{}/**/*.yml", base.to_string_lossy()),
+        format!("{}/match/sub/*.yml", base.to_string_lossy()),
+        // Invalid path
+        "invalid".to_string(),
+      ].iter());
+
+      let mut expected = HashSet::new();
+      expected.insert(format!("{}/match/base.yml", base.to_string_lossy()));
+      expected.insert(format!("{}/match/another.yml", base.to_string_lossy()));
+      expected.insert(format!("{}/match/_sub.yml", base.to_string_lossy()));
+      expected.insert(format!("{}/match/sub/sub.yml", base.to_string_lossy()));
+
+      assert_eq!(result, expected);
+    });
   }
 }

espanso-config/src/config/yaml.rs

@@ -1,6 +1,6 @@
 use anyhow::{private::kind::TraitKind, Result};
 use serde::{Deserialize, Serialize};
-use std::iter::FromIterator;
+use std::{iter::FromIterator, path::Path};
 use std::{collections::HashSet, convert::TryFrom};
 
 use crate::{merge, util::is_yaml_empty};
@@ -124,27 +124,25 @@ impl YAMLConfig {
 
     excludes
   }
-}
 
-// TODO: convert to TryFrom (check the matches section for an example)
+  pub fn generate_match_paths(&self, base_dir: &Path) -> HashSet<String> {
-impl TryFrom<YAMLConfig> for super::Config {
+    let includes = self.aggregate_includes();
-  type Error = anyhow::Error;
+    let excludes = self.aggregate_excludes();
-
-  // TODO: test
-  fn try_from(yaml_config: YAMLConfig) -> Result<Self, Self::Error> {
-    let includes = yaml_config.aggregate_includes();
-    let excludes = yaml_config.aggregate_excludes();
 
     // Extract the paths
-    let exclude_paths = calculate_paths(excludes.iter());
+    let exclude_paths = calculate_paths(base_dir, excludes.iter());
-    let include_paths = calculate_paths(includes.iter());
+    let include_paths = calculate_paths(base_dir, includes.iter());
 
-    let match_files: Vec<String> =
+    HashSet::from_iter(include_paths.difference(&exclude_paths).cloned())
-      Vec::from_iter(include_paths.difference(&exclude_paths).cloned());
+  }
 
-    Ok(Self {
+  // TODO: test
-      label: yaml_config.label,
+  pub fn to_config(&self, base_dir: &Path) -> Result<super::Config> {
-      match_files,
+    let match_paths = self.generate_match_paths(base_dir);
+
+    Ok(super::Config {
+      label: self.label.clone(),
+      match_paths,
     })
   }
 }
@@ -153,15 +151,9 @@ impl TryFrom<YAMLConfig> for super::Config {
 mod tests {
   use super::*;
   use crate::config::Config;
+  use crate::config::path::tests::use_test_directory;
   use std::iter::FromIterator;
   use std::{convert::TryInto, fs::create_dir_all};
-  use tempdir::TempDir;
-
-  // fn create_config(yaml: &str) -> Result<Config> {
-  //   let yaml_config: YAMLConfig = serde_yaml::from_str(yaml)?;
-  //   let m: Config = yaml_config.try_into()?;
-  //   Ok(m)
-  // }
 
   #[test]
   fn aggregate_includes_empty_config() {
@@ -303,5 +295,62 @@ mod tests {
     assert_eq!(child.use_standard_includes, Some(false));
   }
 
+  #[test]
+  fn generate_match_paths_works_correctly() {
+    use_test_directory(|base, match_dir, _| {
+      let sub_dir = match_dir.join("sub");
+      create_dir_all(&sub_dir).unwrap();
+
+      std::fs::write(match_dir.join("base.yml"), "test").unwrap();
+      std::fs::write(match_dir.join("another.yml"), "test").unwrap();
+      std::fs::write(match_dir.join("_sub.yml"), "test").unwrap();
+      std::fs::write(sub_dir.join("sub.yml"), "test").unwrap();
+
+      let config = YAMLConfig::parse_from_str("").unwrap();
+
+      let mut expected = HashSet::new();
+      expected.insert(format!("{}/match/base.yml", base.to_string_lossy()));
+      expected.insert(format!("{}/match/another.yml", base.to_string_lossy()));
+      expected.insert(format!("{}/match/sub/sub.yml", base.to_string_lossy()));
+
+      assert_eq!(config.generate_match_paths(base), expected);
+    });
+  }
+
+  #[test]
+  fn generate_match_paths_works_correctly_with_child_config() {
+    use_test_directory(|base, match_dir, _| {
+      let sub_dir = match_dir.join("sub");
+      create_dir_all(&sub_dir).unwrap();
+
+      std::fs::write(match_dir.join("base.yml"), "test").unwrap();
+      std::fs::write(match_dir.join("another.yml"), "test").unwrap();
+      std::fs::write(match_dir.join("_sub.yml"), "test").unwrap();
+      std::fs::write(sub_dir.join("another.yml"), "test").unwrap();
+      std::fs::write(sub_dir.join("_sub.yml"), "test").unwrap();
+
+      let parent = YAMLConfig::parse_from_str(r"
+      excludes: ['**/another.yml']
+      ").unwrap();
+      let mut child = YAMLConfig::parse_from_str(r"
+      use_standard_includes: false
+      excludes: []
+      includes: ['match/sub/*.yml']
+      ").unwrap();
+      child.merge_parent(&parent);
+
+      let mut expected = HashSet::new();
+      expected.insert(format!("{}/match/sub/another.yml", base.to_string_lossy()));
+      expected.insert(format!("{}/match/sub/_sub.yml", base.to_string_lossy()));
+
+      assert_eq!(child.generate_match_paths(base), expected);
+
+      let mut expected = HashSet::new();
+      expected.insert(format!("{}/match/base.yml", base.to_string_lossy()));
+
+      assert_eq!(parent.generate_match_paths(base), expected);
+    });
+  }
+
   // TODO: test conversion to Config (we need to test that the file match resolution works)
 }

espanso-config/src/counter.rs

@@ -0,0 +1,13 @@
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+static STRUCT_COUNTER: AtomicUsize = AtomicUsize::new(0);
+
+pub type StructId = usize;
+
+/// For performance reasons, some structs need a unique id to be
+/// compared efficiently with one another.
+/// In order to generate it, we use an atomic static variable
+/// that is incremented for each struct.
+pub fn next_id() -> StructId {
+  STRUCT_COUNTER.fetch_add(1, Ordering::SeqCst)
+}

espanso-config/src/lib.rs

@@ -16,10 +16,13 @@
  * You should have received a copy of the GNU General Public License
  * along with espanso.  If not, see <https://www.gnu.org/licenses/>.
  */
+#[macro_use]
+extern crate lazy_static;
 
 mod util;
 mod config;
 mod matches;
+mod counter;
 
 use std::path::Path;
 use anyhow::Result;

espanso-config/src/matches/group/mod.rs

@@ -0,0 +1,133 @@
+use anyhow::Result;
+use log::error;
+use std::{
+  cell::RefCell,
+  convert::TryInto,
+  path::{Path, PathBuf},
+};
+use thiserror::Error;
+
+use super::{Match, Variable};
+
+mod yaml;
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) struct MatchGroup {
+  imports: Vec<String>,
+  pub global_vars: Vec<Variable>,
+  pub matches: Vec<Match>,
+
+  pub resolved_imports: Vec<String>,
+}
+
+impl Default for MatchGroup {
+  fn default() -> Self {
+    Self {
+      imports: Vec::new(),
+      global_vars: Vec::new(),
+      matches: Vec::new(),
+      resolved_imports: Vec::new(),
+    }
+  }
+}
+
+impl MatchGroup {
+  // TODO: test
+  pub fn load(group_path: &Path) -> Result<Self> {
+    if let Some(extension) = group_path.extension() {
+      let extension = extension.to_string_lossy().to_lowercase();
+
+      if extension == "yml" || extension == "yaml" {
+        match yaml::YAMLMatchGroup::parse_from_file(group_path) {
+          Ok(yaml_group) => {
+            let match_group: Result<MatchGroup, _> = yaml_group.try_into();
+            match match_group {
+              Ok(mut group) => {
+                group.resolve_imports(group_path)?;
+                Ok(group)
+              }
+              Err(err) => Err(MatchGroupError::ParsingError(err).into()),
+            }
+          }
+          Err(err) => Err(MatchGroupError::ParsingError(err).into()),
+        }
+      } else {
+        Err(MatchGroupError::InvalidFormat().into())
+      }
+    } else {
+      Err(MatchGroupError::MissingExtension().into())
+    }
+  }
+
+  // TODO: test
+  fn resolve_imports(&mut self, group_path: &Path) -> Result<()> {
+    let mut paths = Vec::new();
+
+    if !group_path.exists() {
+      return Err(
+        MatchGroupError::ResolveImportFailed(format!(
+          "unable to resolve imports for match group at path: {:?}",
+          group_path
+        ))
+        .into(),
+      );
+    }
+
+    // Get the containing directory
+    let current_dir = if group_path.is_file() {
+      if let Some(parent) = group_path.parent() {
+        parent
+      } else {
+        return Err(
+          MatchGroupError::ResolveImportFailed(format!(
+            "unable to resolve imports for match group starting from current path: {:?}",
+            group_path
+          ))
+          .into(),
+        );
+      }
+    } else {
+      group_path
+    };
+
+    for import in self.imports.iter() {
+      let import_path = PathBuf::from(import);
+
+      // Absolute or relative import
+      let full_path = if import_path.is_relative() {
+        current_dir.join(import_path)
+      } else {
+        import_path
+      };
+
+      if full_path.exists() && full_path.is_file() {
+        paths.push(full_path)
+      } else {
+        // Best effort imports
+        error!("unable to resolve import at path: {:?}", full_path);
+      }
+    }
+
+    let string_paths = paths
+      .into_iter()
+      .map(|path| path.to_string_lossy().to_string())
+      .collect();
+    self.resolved_imports = string_paths;
+
+    Ok(())
+  }
+}
+
+#[derive(Error, Debug)]
+pub enum MatchGroupError {
+  #[error("missing extension in match group file")]
+  MissingExtension(),
+
+  #[error("invalid match group format")]
+  InvalidFormat(),
+
+  #[error("parser reported an error: `{0}`")]
+  ParsingError(anyhow::Error),
+
+  #[error("resolve import failed: `{0}`")]
+  ResolveImportFailed(String),
+}

espanso-config/src/matches/group/yaml.rs

@@ -1,17 +1,80 @@
-use std::{
+use std::{collections::HashMap, convert::{TryFrom, TryInto}, path::Path};
-  collections::HashMap,
-  convert::{TryFrom, TryInto},
-};
 
 use anyhow::Result;
 use serde::{Deserialize, Serialize};
 use serde_yaml::{Mapping, Value};
 use thiserror::Error;
 
-use super::{MatchCause, MatchEffect, TextEffect, TriggerCause, Variable};
+use crate::util::is_yaml_empty;
+
+use crate::matches::{Match, MatchCause, MatchEffect, TextEffect, TriggerCause, Variable};
+use super::{MatchGroup};
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
-struct YAMLMatch {
+pub struct YAMLMatchGroup {
+  #[serde(default)]
+  pub imports: Option<Vec<String>>,
+
+  #[serde(default)]
+  pub global_vars: Option<Vec<YAMLVariable>>,
+
+  #[serde(default)]
+  pub matches: Option<Vec<YAMLMatch>>,
+}
+
+impl YAMLMatchGroup {
+  pub fn parse_from_str(yaml: &str) -> Result<Self> {
+    // Because an empty string is not valid YAML but we want to support it anyway
+    if is_yaml_empty(yaml) {
+      return Ok(serde_yaml::from_str(
+        "arbitrary_field_that_will_not_block_the_parser: true",
+      )?);
+    }
+
+    Ok(serde_yaml::from_str(yaml)?)
+  }
+
+  // TODO: test
+  pub fn parse_from_file(path: &Path) -> Result<Self> {
+    let content = std::fs::read_to_string(path)?;
+    Self::parse_from_str(&content)
+  }
+}
+
+impl TryFrom<YAMLMatchGroup> for MatchGroup {
+  type Error = anyhow::Error;
+
+  // TODO: test
+  fn try_from(yaml_match_group: YAMLMatchGroup) -> Result<Self, Self::Error> {
+    let global_vars: Result<Vec<Variable>> = yaml_match_group
+      .global_vars
+      .as_ref()
+      .cloned()
+      .unwrap_or_default()
+      .iter()
+      .map(|var| var.clone().try_into())
+      .collect();
+
+    let matches: Result<Vec<Match>> = yaml_match_group
+      .matches
+      .as_ref()
+      .cloned()
+      .unwrap_or_default()
+      .iter()
+      .map(|m| m.clone().try_into())
+      .collect();
+
+    Ok(MatchGroup {
+      imports: yaml_match_group.imports.unwrap_or_default(),
+      global_vars: global_vars?,
+      matches: matches?,
+      ..Default::default()
+    })
+  }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct YAMLMatch {
   #[serde(default)]
   pub trigger: Option<String>,
 
@@ -73,7 +136,7 @@ fn default_params() -> Mapping {
   Mapping::new()
 }
 
-impl TryFrom<YAMLMatch> for super::Match {
+impl TryFrom<YAMLMatch> for Match {
   type Error = anyhow::Error;
 
   // TODO: test
@@ -126,11 +189,12 @@ impl TryFrom<YAMLMatch> for super::Match {
       cause,
       effect,
       label: None,
+      ..Default::default()
     })
   }
 }
 
-impl TryFrom<YAMLVariable> for super::Variable {
+impl TryFrom<YAMLVariable> for Variable {
   type Error = anyhow::Error;
 
   // TODO: test
@@ -139,6 +203,7 @@ impl TryFrom<YAMLVariable> for super::Variable {
       name: yaml_var.name,
       var_type: yaml_var.var_type,
       params: yaml_var.params,
+      ..Default::default()
     })
   }
 }
@@ -322,6 +387,7 @@ mod tests {
       name: "var1".to_string(),
       var_type: "test".to_string(),
       params,
+      ..Default::default()
     }];
     assert_eq!(
       create_match(
@@ -356,6 +422,7 @@ mod tests {
       name: "var1".to_string(),
       var_type: "test".to_string(),
       params: Mapping::new(),
+      ..Default::default()
     }];
     assert_eq!(
       create_match(

espanso-config/src/matches/mod.rs

@@ -1,6 +1,9 @@
 use serde_yaml::Mapping;
 
-mod yaml;
+use crate::counter::{next_id, StructId};
+
+mod group;
+mod store;
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct Match {
@@ -9,6 +12,9 @@ pub struct Match {
 
   // Metadata
   label: Option<String>,
+
+  // Internals
+  _id: StructId,
 }
 
 impl Default for Match {
@@ -17,6 +23,7 @@ impl Default for Match {
       cause: MatchCause::None,
       effect: MatchEffect::None,
       label: None,
+      _id: next_id(),
     }
   }
 }
@@ -82,4 +89,18 @@ pub struct Variable {
   pub name: String,
   pub var_type: String,
   pub params: Mapping,
+
+  // Internals
+  _id: StructId,
+}
+
+impl Default for Variable {
+  fn default() -> Self {
+    Self {
+      name: String::new(),
+      var_type: String::new(),
+      params: Mapping::new(),
+      _id: next_id(),
+    }
+  }
 }

espanso-config/src/matches/store/default.rs

@@ -0,0 +1,92 @@
+use log::error;
+use std::{
+  collections::{HashMap, HashSet},
+  path::PathBuf,
+};
+
+use super::MatchStore;
+use crate::{counter::StructId, matches::{group::MatchGroup, Match, Variable}};
+
+// TODO: implement store according to notes
+pub(crate) struct DefaultMatchStore {
+  pub groups: HashMap<String, MatchGroup>,
+}
+
+impl DefaultMatchStore {
+  pub fn new() -> Self {
+    Self {
+      groups: HashMap::new(),
+    }
+  }
+}
+
+impl MatchStore for DefaultMatchStore {
+  // TODO: test
+  // TODO: test cyclical imports
+  fn load(&mut self, paths: &[String]) {
+    // Because match groups can imports other match groups,
+    // we have to load them recursively starting from the
+    // top-level ones.
+    load_match_groups_recursively(&mut self.groups, paths);
+  }
+
+  // TODO: test
+  // TODO: test for cyclical imports
+  fn query_set(&self, paths: &[String]) -> super::MatchSet {
+    let mut matches: Vec<&Match> = Vec::new();
+    let mut global_vars: Vec<&Variable> = Vec::new();
+
+    todo!()
+  }
+}
+
+fn load_match_groups_recursively(groups: &mut HashMap<String, MatchGroup>, paths: &[String]) {
+  for path in paths.iter() {
+    if !groups.contains_key(path) {
+      let group_path = PathBuf::from(path);
+      match MatchGroup::load(&group_path) {
+        Ok(group) => {
+          load_match_groups_recursively(groups, &group.resolved_imports);
+          groups.insert(path.clone(), group);
+        },
+        Err(error) => {
+          error!("unable to load match group: {:?}", error);
+        }
+      }
+    }
+  }
+}
+
+fn query_matches_for_paths<'a>(
+  groups: &'a mut HashMap<String, MatchGroup>,
+  visited_paths: &mut HashSet<String>,
+  visited_matches: &mut HashSet<StructId>,
+  visited_global_vars: &mut HashSet<StructId>,
+  matches: &mut Vec<&'a Match>,
+  global_vars: &mut Vec<&'a Variable>,
+  paths: &[String],
+) {
+  for path in paths.iter() {
+    if !visited_paths.contains(path) {
+      if let Some(group) = groups.get(path) {
+        for m in group.matches.iter() {
+          if !visited_matches.contains(&m._id) {
+            matches.push(m);
+            visited_matches.insert(m._id);
+          }
+        }
+
+        for var in group.global_vars.iter() {
+          if !visited_global_vars.contains(&var._id) {
+            global_vars.push(var);
+            visited_global_vars.insert(var._id);
+          }
+        }
+
+        // TODO: here we should visit the imported paths recursively
+      }
+
+      visited_paths.insert(path.clone());
+    }
+  }
+}

espanso-config/src/matches/store/mod.rs

@@ -0,0 +1,18 @@
+use super::{Match, Variable};
+
+mod default;
+
+pub trait MatchStore {
+  fn load(&mut self, paths: &[String]);
+  fn query_set(&self, paths: &[String]) -> MatchSet;
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct MatchSet<'a> {
+  pub matches: Vec<&'a Match>,
+  pub global_vars: Vec<&'a Variable>,
+}
+
+pub fn new() -> impl MatchStore {
+  default::DefaultMatchStore::new()
+}