feat(package): progress in package archiver implementation

espanso-package/Cargo.toml

@@ -17,4 +17,6 @@ natord = "1.0.9"
 reqwest = { version = "0.11.4", features = ["blocking"] }
 lazy_static = "1.4.0"
 regex = "1.4.3"
-zip = "0.5.13"
+zip = "0.5.13"
+scopeguard = "1.1.0"
+fs_extra = "1.2.0"

espanso-package/src/archive/default.rs

@@ -0,0 +1,109 @@
+/*
+ * This file is part of espanso.
+ *
+ * Copyright (C) 2019-2021 Federico Terzi
+ *
+ * espanso is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * espanso is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with espanso.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+use anyhow::{bail, Context, Result};
+use std::path::{Path, PathBuf};
+
+use crate::{ArchivedPackage, Archiver, Package, PackageSpecifier, SaveOptions};
+
+use super::StoredPackage;
+
+pub struct DefaultArchiver {
+  package_dir: PathBuf,
+}
+
+impl DefaultArchiver {
+  pub fn new(package_dir: &Path) -> Self {
+    Self {
+      package_dir: package_dir.to_owned(),
+    }
+  }
+}
+
+impl Archiver for DefaultArchiver {
+  fn save(
+    &self,
+    package: &dyn Package,
+    specifier: &PackageSpecifier,
+    save_options: &SaveOptions,
+  ) -> Result<ArchivedPackage> {
+    let target_dir = self.package_dir.join(package.name());
+
+    if target_dir.is_dir() && !save_options.overwrite_existing {
+      bail!("package {} is already installed", package.name());
+    }
+
+    // Backup the previous directory if present
+    let backup_dir = self.package_dir.join(&format!("{}.old", package.name()));
+    let _backup_guard = if target_dir.is_dir() {
+      std::fs::rename(&target_dir, &backup_dir)
+        .context("unable to backup old package directory")?;
+
+      // If the function returns due to an error, restore the previous directory
+      Some(scopeguard::guard(
+        (backup_dir.clone(), target_dir.clone()),
+        |(backup_dir, target_dir)| {
+          if backup_dir.is_dir() {
+            if target_dir.is_dir() {
+              std::fs::remove_dir_all(&target_dir)
+                .expect("unable to remove dirty package directory");
+            }
+
+            std::fs::rename(backup_dir, target_dir).expect("unable to restore backup directory");
+          }
+        },
+      ))
+    } else {
+      None
+    };
+
+    std::fs::create_dir_all(&target_dir).context("unable to create target directory")?;
+
+    super::util::copy_dir_without_dot_files(package.location(), &target_dir)
+      .context("unable to copy package files")?;
+
+    super::util::create_package_source_file(specifier, &target_dir)
+      .context("unable to create _pkgsource.yml file")?;
+
+    // Remove backup
+    if backup_dir.is_dir() {
+      std::fs::remove_dir_all(backup_dir).context("unable to remove backup directory")?;
+    }
+
+    let archived_package =
+      super::read::read_archived_package(&target_dir).context("unable to load archived package")?;
+
+    Ok(archived_package)
+  }
+
+  fn get(&self, name: &str) -> Result<ArchivedPackage> {
+    todo!()
+  }
+
+  fn list(&self) -> Result<Vec<StoredPackage>> {
+    todo!()
+  }
+
+  fn delete(&self, name: &str) -> Result<()> {
+    todo!()
+  }
+}
+
+// TODO: test
+// TODO: test what happens with "legacy" packages

espanso-package/src/archive/mod.rs

@@ -0,0 +1,108 @@
+/*
+ * This file is part of espanso.
+ *
+ * Copyright (C) 2019-2021 Federico Terzi
+ *
+ * espanso is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * espanso is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with espanso.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+
+use crate::{Package, PackageSpecifier, manifest::Manifest};
+
+pub mod default;
+mod read;
+mod util;
+
+pub const PACKAGE_SOURCE_FILE: &str = "_pkgsource.yml";
+
+#[derive(Debug)]
+pub struct ArchivedPackage {
+  // Metadata
+  pub manifest: Manifest,
+
+  // Package source information (needed to update)
+  pub source: PackageSource,
+}
+
+#[derive(Debug)]
+pub struct LegacyPackage {
+  pub name: String,
+}
+
+#[derive(Debug)]
+pub enum StoredPackage {
+  Legacy(LegacyPackage),
+  Modern(ArchivedPackage),
+}
+
+pub trait Archiver {
+  fn get(&self, name: &str) -> Result<ArchivedPackage>;
+  fn save(&self, package: &dyn Package, specifier: &PackageSpecifier, save_options: &SaveOptions) -> Result<ArchivedPackage>;
+  fn list(&self) -> Result<Vec<StoredPackage>>;
+  fn delete(&self, name: &str) -> Result<()>;
+}
+
+#[derive(Debug, Default)]
+pub struct SaveOptions {
+  pub overwrite_existing: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum PackageSource {
+  Hub,
+  Git {
+    repo_url: String,
+    repo_branch: Option<String>,
+    use_native_git: bool,
+  },
+}
+
+impl From<&PackageSpecifier> for PackageSource {
+  fn from(package: &PackageSpecifier) -> Self {
+    if let Some(git_repo_url) = package.git_repo_url.as_deref() {
+      Self::Git {
+        repo_url: git_repo_url.to_string(),
+        repo_branch: package.git_branch.clone(),
+        use_native_git: package.use_native_git,
+      }
+    } else {
+      Self::Hub
+    }
+  }
+}
+
+impl From<&ArchivedPackage> for PackageSpecifier {
+  fn from(package: &ArchivedPackage) -> Self {
+    match &package.source {
+      PackageSource::Hub => PackageSpecifier {
+        name: package.manifest.name.to_string(),
+        ..Default::default()
+      },
+      PackageSource::Git {
+        repo_url,
+        repo_branch,
+        use_native_git,
+      } => PackageSpecifier {
+        name: package.manifest.name.to_string(),
+        git_repo_url: Some(repo_url.to_string()),
+        git_branch: repo_branch.clone(),
+        use_native_git: *use_native_git,
+        ..Default::default()
+      },
+    }
+  }
+}

espanso-package/src/archive/read.rs

@@ -0,0 +1,48 @@
+/*
+ * This file is part of espanso.
+ *
+ * Copyright (C) 2019-2021 Federico Terzi
+ *
+ * espanso is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * espanso is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with espanso.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+use std::path::Path;
+
+use anyhow::{bail, Context, Result};
+
+use crate::{manifest::Manifest, ArchivedPackage};
+
+use super::{PackageSource, PACKAGE_SOURCE_FILE};
+
+pub fn read_archived_package(containing_dir: &Path) -> Result<ArchivedPackage> {
+  let manifest_path = containing_dir.join("_manifest.yml");
+  if !manifest_path.is_file() {
+    bail!("missing _manifest.yml file");
+  }
+
+  let source_path = containing_dir.join(PACKAGE_SOURCE_FILE);
+  let source = if source_path.is_file() {
+    let yaml = std::fs::read_to_string(&source_path)?;
+    let source: PackageSource =
+      serde_yaml::from_str(&yaml).context("unable to parse package source file.")?;
+    source
+  } else {
+    // Fallback to hub installation
+    PackageSource::Hub
+  };
+
+  let manifest = Manifest::parse(&manifest_path).context("unable to parse manifest file")?;
+
+  Ok(ArchivedPackage { manifest, source })
+}

espanso-package/src/archive/util.rs

@@ -0,0 +1,60 @@
+/*
+ * This file is part of espanso.
+ *
+ * Copyright (C) 2019-2021 Federico Terzi
+ *
+ * espanso is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * espanso is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with espanso.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+use std::path::Path;
+
+use anyhow::Result;
+use fs_extra::dir::CopyOptions;
+
+use crate::PackageSpecifier;
+
+use super::{PACKAGE_SOURCE_FILE, PackageSource};
+
+// TODO: test
+pub fn copy_dir_without_dot_files(source_dir: &Path, inside_dir: &Path) -> Result<()> {
+  fs_extra::dir::copy(
+    source_dir,
+    inside_dir,
+    &CopyOptions {
+      copy_inside: true,
+      content_only: true,
+      ..Default::default()
+    },
+  )?;
+
+  // Remove dot files and dirs (such as .git)
+  let mut to_be_removed = Vec::new();
+  for path in std::fs::read_dir(inside_dir)? {
+    let path = path?.path();
+    if path.starts_with(".") {
+      to_be_removed.push(path);
+    }
+  }
+
+  fs_extra::remove_items(&to_be_removed)?;
+
+  Ok(())
+}
+
+pub fn create_package_source_file(specifier: &PackageSpecifier, target_dir: &Path) -> Result<()> {
+  let source: PackageSource = specifier.into();
+  let yaml = serde_yaml::to_string(&source)?;
+  std::fs::write(target_dir.join(PACKAGE_SOURCE_FILE), yaml)?;
+  Ok(())
+}

espanso-package/src/lib.rs

@@ -19,79 +19,53 @@
 
 use std::path::Path;
 
-use anyhow::{Result, bail};
+use anyhow::{bail, Result};
-use thiserror::Error;
 
+mod archive;
 mod manifest;
 mod package;
 mod provider;
 mod resolver;
 mod util;
 
-#[derive(Debug, Default)]
+pub use archive::{ArchivedPackage, Archiver, SaveOptions};
-pub struct PackageSpecifier {
+pub use provider::{PackageSpecifier, PackageProvider};
-  pub name: String,
+pub use package::Package;
-  pub version: Option<String>,
-
-  // Source information
-  pub git_repo_url: Option<String>,
-  pub git_branch: Option<String>,
-}
-
-pub trait Package {
-  // Manifest
-  fn name(&self) -> &str;
-  fn title(&self) -> &str;
-  fn description(&self) -> &str;
-  fn version(&self) -> &str;
-  fn author(&self) -> &str;
-
-  // Directory containing the package files
-  fn location(&self) -> &Path;
-}
-
-pub trait PackageProvider {
-  fn download(&self, package: &PackageSpecifier) -> Result<Box<dyn Package>>;
-  // TODO: fn check update available? (probably should be only available in the hub)
-}
 
 // TODO: once the download is completed, avoid copying files beginning with "."
 
-#[derive(Error, Debug)]
-pub enum PackageResolutionError {
-  #[error("package not found")]
-  PackageNotFound,
-}
-
 pub fn get_provider(package: &PackageSpecifier) -> Result<Box<dyn PackageProvider>> {
   if let Some(git_repo_url) = package.git_repo_url.as_deref() {
-    let matches_known_hosts = if let Some(github_parts) = util::github::extract_github_url_parts(git_repo_url) {
+    if !package.use_native_git {
-      if let Some(repo_scheme) =
+      let matches_known_hosts =
-        util::github::resolve_repo_scheme(github_parts, package.git_branch.as_deref())?
+        if let Some(github_parts) = util::github::extract_github_url_parts(git_repo_url) {
-      {
+          if let Some(repo_scheme) =
-        return Ok(Box::new(provider::github::GitHubPackageProvider::new(
+            util::github::resolve_repo_scheme(github_parts, package.git_branch.as_deref())?
-          repo_scheme.author,
+          {
-          repo_scheme.name,
+            return Ok(Box::new(provider::github::GitHubPackageProvider::new(
-          repo_scheme.branch,
+              repo_scheme.author,
-        )));
+              repo_scheme.name,
+              repo_scheme.branch,
+            )));
+          }
+
+          true
+        } else if let Some(gitlab_parts) = util::gitlab::extract_gitlab_url_parts(git_repo_url) {
+          panic!("GitLab is not supported yet!");
+          todo!();
+
+          true
+        } else {
+          false
+        };
+
+      // Git repository seems to be in one of the known hosts, but the direct methods
+      // couldn't retrieve its content. This might happen with private repos (as they are not
+      // available to non-authenticated requests), so we check if a "git ls-remote" command
+      // is able to access it.
+      if matches_known_hosts && !util::git::is_private_repo(git_repo_url) {
+        bail!("could not access repository: {}, make sure it exists and that you have the necessary access rights.");
       }
-
-      true
-    } else if let Some(gitlab_parts) = util::gitlab::extract_gitlab_url_parts(git_repo_url) {
-      panic!("GitLab is not supported yet!");
-      todo!();
-
-      true
-    } else {
-      false
-    };
-
-    // Git repository seems to be in one of the known hosts, but the direct methods
-    // couldn't retrieve its content. This might happen with private repos (as they are not
-    // available to non-authenticated requests), so we check if a "git ls-remote" command 
-    // is able to access it.
-    if matches_known_hosts && !util::git::is_private_repo(git_repo_url) {
-      bail!("could not access repository: {}, make sure it exists and that you have the necessary access rights.");
     }
 
     // Git repository is neither on Github or Gitlab
@@ -103,3 +77,8 @@ pub fn get_provider(package: &PackageSpecifier) -> Result<Box<dyn PackageProvide
     todo!();
   }
 }
+
+
+pub fn get_archiver(package_dir: &Path) -> Result<Box<dyn Archiver>> {
+  Ok(Box::new(archive::default::DefaultArchiver::new(package_dir)))
+}

espanso-package/src/package/mod.rs

@@ -0,0 +1,51 @@
+/*
+ * This file is part of espanso.
+ *
+ * Copyright (C) 2019-2021 Federico Terzi
+ *
+ * espanso is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * espanso is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with espanso.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+use std::path::Path;
+
+pub mod default;
+
+pub trait Package {
+  // Manifest
+  fn name(&self) -> &str;
+  fn title(&self) -> &str;
+  fn description(&self) -> &str;
+  fn version(&self) -> &str;
+  fn author(&self) -> &str;
+
+  // Directory containing the package files
+  fn location(&self) -> &Path;
+}
+
+impl std::fmt::Debug for dyn Package {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    writeln!(
+      f,
+      "name: {}\nversion: {}\ntitle: {}\ndescription: {}\nauthor: {}\nlocation: {:?}",
+      self.name(),
+      self.version(),
+      self.title(),
+      self.description(),
+      self.author(),
+      self.location()
+    )
+  }
+}
+
+pub use default::DefaultPackage;

espanso-package/src/provider/git.rs

@@ -20,10 +20,11 @@
 use crate::{
   package::DefaultPackage,
   resolver::{resolve_package},
-  Package, PackageProvider, PackageSpecifier,
+  Package, PackageSpecifier,
 };
 use anyhow::{anyhow, bail, Context, Result};
 use std::{path::Path, process::Command};
+use super::PackageProvider;
 
 pub struct GitPackageProvider {}
 
@@ -72,6 +73,10 @@ impl GitPackageProvider {
 }
 
 impl PackageProvider for GitPackageProvider {
+  fn name(&self) -> String {
+    "git".to_string()
+  }
+
   fn download(&self, package: &PackageSpecifier) -> Result<Box<dyn Package>> {
     if !Self::is_git_installed() {
       bail!("unable to invoke 'git' command, please make sure it is installed and visible in PATH");

espanso-package/src/provider/github.rs

@@ -18,9 +18,10 @@
  */
 
 use crate::{
-  package::DefaultPackage, resolver::resolve_package, Package, PackageProvider, PackageSpecifier,
+  package::DefaultPackage, resolver::resolve_package, Package, PackageSpecifier,
 };
 use anyhow::{Result};
+use super::PackageProvider;
 
 pub struct GitHubPackageProvider {
   repo_author: String,
@@ -39,6 +40,10 @@ impl GitHubPackageProvider {
 }
 
 impl PackageProvider for GitHubPackageProvider {
+  fn name(&self) -> String {
+    "github".to_string()
+  }
+
   fn download(&self, package: &PackageSpecifier) -> Result<Box<dyn Package>> {
     let download_url = format!(
       "https://github.com/{}/{}/archive/refs/heads/{}.zip",
@@ -80,6 +85,7 @@ mod tests {
         version: None,
         git_repo_url: Some("https://github.com/espanso/dummy-package".to_string()),
         git_branch: None,
+        ..Default::default()
       })
       .unwrap();
 

espanso-package/src/provider/mod.rs

@@ -17,5 +17,29 @@
  * along with espanso.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+use anyhow::Result;
+
+use crate::Package;
+
 pub(crate) mod git;
-pub(crate) mod github;
+pub(crate) mod github;
+
+#[derive(Debug, Default)]
+pub struct PackageSpecifier {
+  pub name: String,
+  pub version: Option<String>,
+
+  // Source information
+  pub git_repo_url: Option<String>,
+  pub git_branch: Option<String>,
+
+  // Resolution options
+  pub use_native_git: bool,
+}
+
+pub trait PackageProvider {
+  fn name(&self) -> String;
+  fn download(&self, package: &PackageSpecifier) -> Result<Box<dyn Package>>;
+  // TODO: fn check update available? (probably should be only available in the hub)
+}
+

espanso-package/src/resolver.rs

@@ -59,8 +59,8 @@ pub fn resolve_package(
     Ok(matching_package)
   } else {
     bail!(
-      "unable to find version: {:?} for package: {}",
+      "unable to find version: {} for package: {}",
-      version,
+      version.unwrap_or_default(),
       name
     );
   }

espanso-package/src/util/github.rs

@@ -85,7 +85,7 @@ pub fn check_repo_with_branch(parts: &GitHubParts, branch: &str) -> Result<bool>
   let url = generate_github_download_url(parts, branch);
   let response = client.head(url).send()?;
 
-  Ok(response.status() == StatusCode::FOUND)
+  Ok(response.status() == StatusCode::FOUND || response.status() == StatusCode::OK)
 }
 
 fn generate_github_download_url(parts: &GitHubParts, branch: &str) -> String {