From 5714ebe131babff3ec5d8721098280b7fd6da79d Mon Sep 17 00:00:00 2001 From: Federico Terzi Date: Wed, 1 Sep 2021 22:11:28 +0200 Subject: [PATCH] feat(package): progress in package archiver implementation --- espanso-package/Cargo.toml | 4 +- espanso-package/src/archive/default.rs | 109 ++++++++++++++++++ espanso-package/src/archive/mod.rs | 108 +++++++++++++++++ espanso-package/src/archive/read.rs | 48 ++++++++ espanso-package/src/archive/util.rs | 60 ++++++++++ espanso-package/src/lib.rs | 99 +++++++--------- .../src/{package.rs => package/default.rs} | 0 espanso-package/src/package/mod.rs | 51 ++++++++ espanso-package/src/provider/git.rs | 7 +- espanso-package/src/provider/github.rs | 8 +- espanso-package/src/provider/mod.rs | 26 ++++- espanso-package/src/resolver.rs | 4 +- espanso-package/src/util/github.rs | 2 +- 13 files changed, 459 insertions(+), 67 deletions(-) create mode 100644 espanso-package/src/archive/default.rs create mode 100644 espanso-package/src/archive/mod.rs create mode 100644 espanso-package/src/archive/read.rs create mode 100644 espanso-package/src/archive/util.rs rename espanso-package/src/{package.rs => package/default.rs} (100%) create mode 100644 espanso-package/src/package/mod.rs diff --git a/espanso-package/Cargo.toml b/espanso-package/Cargo.toml index 87e7525..72b1d42 100644 --- a/espanso-package/Cargo.toml +++ b/espanso-package/Cargo.toml @@ -17,4 +17,6 @@ natord = "1.0.9" reqwest = { version = "0.11.4", features = ["blocking"] } lazy_static = "1.4.0" regex = "1.4.3" -zip = "0.5.13" \ No newline at end of file +zip = "0.5.13" +scopeguard = "1.1.0" +fs_extra = "1.2.0" \ No newline at end of file diff --git a/espanso-package/src/archive/default.rs b/espanso-package/src/archive/default.rs new file mode 100644 index 0000000..882e30f --- /dev/null +++ b/espanso-package/src/archive/default.rs @@ -0,0 +1,109 @@ +/* + * This file is part of espanso. + * + * Copyright (C) 2019-2021 Federico Terzi + * + * espanso is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * espanso is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with espanso. If not, see . + */ + +use anyhow::{bail, Context, Result}; +use std::path::{Path, PathBuf}; + +use crate::{ArchivedPackage, Archiver, Package, PackageSpecifier, SaveOptions}; + +use super::StoredPackage; + +pub struct DefaultArchiver { + package_dir: PathBuf, +} + +impl DefaultArchiver { + pub fn new(package_dir: &Path) -> Self { + Self { + package_dir: package_dir.to_owned(), + } + } +} + +impl Archiver for DefaultArchiver { + fn save( + &self, + package: &dyn Package, + specifier: &PackageSpecifier, + save_options: &SaveOptions, + ) -> Result { + let target_dir = self.package_dir.join(package.name()); + + if target_dir.is_dir() && !save_options.overwrite_existing { + bail!("package {} is already installed", package.name()); + } + + // Backup the previous directory if present + let backup_dir = self.package_dir.join(&format!("{}.old", package.name())); + let _backup_guard = if target_dir.is_dir() { + std::fs::rename(&target_dir, &backup_dir) + .context("unable to backup old package directory")?; + + // If the function returns due to an error, restore the previous directory + Some(scopeguard::guard( + (backup_dir.clone(), target_dir.clone()), + |(backup_dir, target_dir)| { + if backup_dir.is_dir() { + if target_dir.is_dir() { + std::fs::remove_dir_all(&target_dir) + .expect("unable to remove dirty package directory"); + } + + std::fs::rename(backup_dir, target_dir).expect("unable to restore backup directory"); + } + }, + )) + } else { + None + }; + + std::fs::create_dir_all(&target_dir).context("unable to create target directory")?; + + super::util::copy_dir_without_dot_files(package.location(), &target_dir) + .context("unable to copy package files")?; + + super::util::create_package_source_file(specifier, &target_dir) + .context("unable to create _pkgsource.yml file")?; + + // Remove backup + if backup_dir.is_dir() { + std::fs::remove_dir_all(backup_dir).context("unable to remove backup directory")?; + } + + let archived_package = + super::read::read_archived_package(&target_dir).context("unable to load archived package")?; + + Ok(archived_package) + } + + fn get(&self, name: &str) -> Result { + todo!() + } + + fn list(&self) -> Result> { + todo!() + } + + fn delete(&self, name: &str) -> Result<()> { + todo!() + } +} + +// TODO: test +// TODO: test what happens with "legacy" packages diff --git a/espanso-package/src/archive/mod.rs b/espanso-package/src/archive/mod.rs new file mode 100644 index 0000000..8be0557 --- /dev/null +++ b/espanso-package/src/archive/mod.rs @@ -0,0 +1,108 @@ +/* + * This file is part of espanso. + * + * Copyright (C) 2019-2021 Federico Terzi + * + * espanso is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * espanso is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with espanso. If not, see . + */ + +use anyhow::Result; +use serde::{Deserialize, Serialize}; + +use crate::{Package, PackageSpecifier, manifest::Manifest}; + +pub mod default; +mod read; +mod util; + +pub const PACKAGE_SOURCE_FILE: &str = "_pkgsource.yml"; + +#[derive(Debug)] +pub struct ArchivedPackage { + // Metadata + pub manifest: Manifest, + + // Package source information (needed to update) + pub source: PackageSource, +} + +#[derive(Debug)] +pub struct LegacyPackage { + pub name: String, +} + +#[derive(Debug)] +pub enum StoredPackage { + Legacy(LegacyPackage), + Modern(ArchivedPackage), +} + +pub trait Archiver { + fn get(&self, name: &str) -> Result; + fn save(&self, package: &dyn Package, specifier: &PackageSpecifier, save_options: &SaveOptions) -> Result; + fn list(&self) -> Result>; + fn delete(&self, name: &str) -> Result<()>; +} + +#[derive(Debug, Default)] +pub struct SaveOptions { + pub overwrite_existing: bool, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PackageSource { + Hub, + Git { + repo_url: String, + repo_branch: Option, + use_native_git: bool, + }, +} + +impl From<&PackageSpecifier> for PackageSource { + fn from(package: &PackageSpecifier) -> Self { + if let Some(git_repo_url) = package.git_repo_url.as_deref() { + Self::Git { + repo_url: git_repo_url.to_string(), + repo_branch: package.git_branch.clone(), + use_native_git: package.use_native_git, + } + } else { + Self::Hub + } + } +} + +impl From<&ArchivedPackage> for PackageSpecifier { + fn from(package: &ArchivedPackage) -> Self { + match &package.source { + PackageSource::Hub => PackageSpecifier { + name: package.manifest.name.to_string(), + ..Default::default() + }, + PackageSource::Git { + repo_url, + repo_branch, + use_native_git, + } => PackageSpecifier { + name: package.manifest.name.to_string(), + git_repo_url: Some(repo_url.to_string()), + git_branch: repo_branch.clone(), + use_native_git: *use_native_git, + ..Default::default() + }, + } + } +} diff --git a/espanso-package/src/archive/read.rs b/espanso-package/src/archive/read.rs new file mode 100644 index 0000000..dfd2b97 --- /dev/null +++ b/espanso-package/src/archive/read.rs @@ -0,0 +1,48 @@ +/* + * This file is part of espanso. + * + * Copyright (C) 2019-2021 Federico Terzi + * + * espanso is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * espanso is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with espanso. If not, see . + */ + +use std::path::Path; + +use anyhow::{bail, Context, Result}; + +use crate::{manifest::Manifest, ArchivedPackage}; + +use super::{PackageSource, PACKAGE_SOURCE_FILE}; + +pub fn read_archived_package(containing_dir: &Path) -> Result { + let manifest_path = containing_dir.join("_manifest.yml"); + if !manifest_path.is_file() { + bail!("missing _manifest.yml file"); + } + + let source_path = containing_dir.join(PACKAGE_SOURCE_FILE); + let source = if source_path.is_file() { + let yaml = std::fs::read_to_string(&source_path)?; + let source: PackageSource = + serde_yaml::from_str(&yaml).context("unable to parse package source file.")?; + source + } else { + // Fallback to hub installation + PackageSource::Hub + }; + + let manifest = Manifest::parse(&manifest_path).context("unable to parse manifest file")?; + + Ok(ArchivedPackage { manifest, source }) +} diff --git a/espanso-package/src/archive/util.rs b/espanso-package/src/archive/util.rs new file mode 100644 index 0000000..8d07613 --- /dev/null +++ b/espanso-package/src/archive/util.rs @@ -0,0 +1,60 @@ +/* + * This file is part of espanso. + * + * Copyright (C) 2019-2021 Federico Terzi + * + * espanso is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * espanso is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with espanso. If not, see . + */ + +use std::path::Path; + +use anyhow::Result; +use fs_extra::dir::CopyOptions; + +use crate::PackageSpecifier; + +use super::{PACKAGE_SOURCE_FILE, PackageSource}; + +// TODO: test +pub fn copy_dir_without_dot_files(source_dir: &Path, inside_dir: &Path) -> Result<()> { + fs_extra::dir::copy( + source_dir, + inside_dir, + &CopyOptions { + copy_inside: true, + content_only: true, + ..Default::default() + }, + )?; + + // Remove dot files and dirs (such as .git) + let mut to_be_removed = Vec::new(); + for path in std::fs::read_dir(inside_dir)? { + let path = path?.path(); + if path.starts_with(".") { + to_be_removed.push(path); + } + } + + fs_extra::remove_items(&to_be_removed)?; + + Ok(()) +} + +pub fn create_package_source_file(specifier: &PackageSpecifier, target_dir: &Path) -> Result<()> { + let source: PackageSource = specifier.into(); + let yaml = serde_yaml::to_string(&source)?; + std::fs::write(target_dir.join(PACKAGE_SOURCE_FILE), yaml)?; + Ok(()) +} \ No newline at end of file diff --git a/espanso-package/src/lib.rs b/espanso-package/src/lib.rs index b8aef33..75b67dd 100644 --- a/espanso-package/src/lib.rs +++ b/espanso-package/src/lib.rs @@ -19,79 +19,53 @@ use std::path::Path; -use anyhow::{Result, bail}; -use thiserror::Error; +use anyhow::{bail, Result}; +mod archive; mod manifest; mod package; mod provider; mod resolver; mod util; -#[derive(Debug, Default)] -pub struct PackageSpecifier { - pub name: String, - pub version: Option, - - // Source information - pub git_repo_url: Option, - pub git_branch: Option, -} - -pub trait Package { - // Manifest - fn name(&self) -> &str; - fn title(&self) -> &str; - fn description(&self) -> &str; - fn version(&self) -> &str; - fn author(&self) -> &str; - - // Directory containing the package files - fn location(&self) -> &Path; -} - -pub trait PackageProvider { - fn download(&self, package: &PackageSpecifier) -> Result>; - // TODO: fn check update available? (probably should be only available in the hub) -} +pub use archive::{ArchivedPackage, Archiver, SaveOptions}; +pub use provider::{PackageSpecifier, PackageProvider}; +pub use package::Package; // TODO: once the download is completed, avoid copying files beginning with "." -#[derive(Error, Debug)] -pub enum PackageResolutionError { - #[error("package not found")] - PackageNotFound, -} - pub fn get_provider(package: &PackageSpecifier) -> Result> { if let Some(git_repo_url) = package.git_repo_url.as_deref() { - let matches_known_hosts = if let Some(github_parts) = util::github::extract_github_url_parts(git_repo_url) { - if let Some(repo_scheme) = - util::github::resolve_repo_scheme(github_parts, package.git_branch.as_deref())? - { - return Ok(Box::new(provider::github::GitHubPackageProvider::new( - repo_scheme.author, - repo_scheme.name, - repo_scheme.branch, - ))); + if !package.use_native_git { + let matches_known_hosts = + if let Some(github_parts) = util::github::extract_github_url_parts(git_repo_url) { + if let Some(repo_scheme) = + util::github::resolve_repo_scheme(github_parts, package.git_branch.as_deref())? + { + return Ok(Box::new(provider::github::GitHubPackageProvider::new( + repo_scheme.author, + repo_scheme.name, + repo_scheme.branch, + ))); + } + + true + } else if let Some(gitlab_parts) = util::gitlab::extract_gitlab_url_parts(git_repo_url) { + panic!("GitLab is not supported yet!"); + todo!(); + + true + } else { + false + }; + + // Git repository seems to be in one of the known hosts, but the direct methods + // couldn't retrieve its content. This might happen with private repos (as they are not + // available to non-authenticated requests), so we check if a "git ls-remote" command + // is able to access it. + if matches_known_hosts && !util::git::is_private_repo(git_repo_url) { + bail!("could not access repository: {}, make sure it exists and that you have the necessary access rights."); } - - true - } else if let Some(gitlab_parts) = util::gitlab::extract_gitlab_url_parts(git_repo_url) { - panic!("GitLab is not supported yet!"); - todo!(); - - true - } else { - false - }; - - // Git repository seems to be in one of the known hosts, but the direct methods - // couldn't retrieve its content. This might happen with private repos (as they are not - // available to non-authenticated requests), so we check if a "git ls-remote" command - // is able to access it. - if matches_known_hosts && !util::git::is_private_repo(git_repo_url) { - bail!("could not access repository: {}, make sure it exists and that you have the necessary access rights."); } // Git repository is neither on Github or Gitlab @@ -103,3 +77,8 @@ pub fn get_provider(package: &PackageSpecifier) -> Result Result> { + Ok(Box::new(archive::default::DefaultArchiver::new(package_dir))) +} \ No newline at end of file diff --git a/espanso-package/src/package.rs b/espanso-package/src/package/default.rs similarity index 100% rename from espanso-package/src/package.rs rename to espanso-package/src/package/default.rs diff --git a/espanso-package/src/package/mod.rs b/espanso-package/src/package/mod.rs new file mode 100644 index 0000000..8a45e7e --- /dev/null +++ b/espanso-package/src/package/mod.rs @@ -0,0 +1,51 @@ +/* + * This file is part of espanso. + * + * Copyright (C) 2019-2021 Federico Terzi + * + * espanso is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * espanso is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with espanso. If not, see . + */ + +use std::path::Path; + +pub mod default; + +pub trait Package { + // Manifest + fn name(&self) -> &str; + fn title(&self) -> &str; + fn description(&self) -> &str; + fn version(&self) -> &str; + fn author(&self) -> &str; + + // Directory containing the package files + fn location(&self) -> &Path; +} + +impl std::fmt::Debug for dyn Package { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!( + f, + "name: {}\nversion: {}\ntitle: {}\ndescription: {}\nauthor: {}\nlocation: {:?}", + self.name(), + self.version(), + self.title(), + self.description(), + self.author(), + self.location() + ) + } +} + +pub use default::DefaultPackage; \ No newline at end of file diff --git a/espanso-package/src/provider/git.rs b/espanso-package/src/provider/git.rs index 68f111f..27e8eef 100644 --- a/espanso-package/src/provider/git.rs +++ b/espanso-package/src/provider/git.rs @@ -20,10 +20,11 @@ use crate::{ package::DefaultPackage, resolver::{resolve_package}, - Package, PackageProvider, PackageSpecifier, + Package, PackageSpecifier, }; use anyhow::{anyhow, bail, Context, Result}; use std::{path::Path, process::Command}; +use super::PackageProvider; pub struct GitPackageProvider {} @@ -72,6 +73,10 @@ impl GitPackageProvider { } impl PackageProvider for GitPackageProvider { + fn name(&self) -> String { + "git".to_string() + } + fn download(&self, package: &PackageSpecifier) -> Result> { if !Self::is_git_installed() { bail!("unable to invoke 'git' command, please make sure it is installed and visible in PATH"); diff --git a/espanso-package/src/provider/github.rs b/espanso-package/src/provider/github.rs index e863af7..22f70c2 100644 --- a/espanso-package/src/provider/github.rs +++ b/espanso-package/src/provider/github.rs @@ -18,9 +18,10 @@ */ use crate::{ - package::DefaultPackage, resolver::resolve_package, Package, PackageProvider, PackageSpecifier, + package::DefaultPackage, resolver::resolve_package, Package, PackageSpecifier, }; use anyhow::{Result}; +use super::PackageProvider; pub struct GitHubPackageProvider { repo_author: String, @@ -39,6 +40,10 @@ impl GitHubPackageProvider { } impl PackageProvider for GitHubPackageProvider { + fn name(&self) -> String { + "github".to_string() + } + fn download(&self, package: &PackageSpecifier) -> Result> { let download_url = format!( "https://github.com/{}/{}/archive/refs/heads/{}.zip", @@ -80,6 +85,7 @@ mod tests { version: None, git_repo_url: Some("https://github.com/espanso/dummy-package".to_string()), git_branch: None, + ..Default::default() }) .unwrap(); diff --git a/espanso-package/src/provider/mod.rs b/espanso-package/src/provider/mod.rs index 146a2fe..cf4388b 100644 --- a/espanso-package/src/provider/mod.rs +++ b/espanso-package/src/provider/mod.rs @@ -17,5 +17,29 @@ * along with espanso. If not, see . */ +use anyhow::Result; + +use crate::Package; + pub(crate) mod git; -pub(crate) mod github; \ No newline at end of file +pub(crate) mod github; + +#[derive(Debug, Default)] +pub struct PackageSpecifier { + pub name: String, + pub version: Option, + + // Source information + pub git_repo_url: Option, + pub git_branch: Option, + + // Resolution options + pub use_native_git: bool, +} + +pub trait PackageProvider { + fn name(&self) -> String; + fn download(&self, package: &PackageSpecifier) -> Result>; + // TODO: fn check update available? (probably should be only available in the hub) +} + diff --git a/espanso-package/src/resolver.rs b/espanso-package/src/resolver.rs index 380b20d..a7d5c4d 100644 --- a/espanso-package/src/resolver.rs +++ b/espanso-package/src/resolver.rs @@ -59,8 +59,8 @@ pub fn resolve_package( Ok(matching_package) } else { bail!( - "unable to find version: {:?} for package: {}", - version, + "unable to find version: {} for package: {}", + version.unwrap_or_default(), name ); } diff --git a/espanso-package/src/util/github.rs b/espanso-package/src/util/github.rs index 8d1c6d1..7cedcc8 100644 --- a/espanso-package/src/util/github.rs +++ b/espanso-package/src/util/github.rs @@ -85,7 +85,7 @@ pub fn check_repo_with_branch(parts: &GitHubParts, branch: &str) -> Result let url = generate_github_download_url(parts, branch); let response = client.head(url).send()?; - Ok(response.status() == StatusCode::FOUND) + Ok(response.status() == StatusCode::FOUND || response.status() == StatusCode::OK) } fn generate_github_download_url(parts: &GitHubParts, branch: &str) -> String {