feat(*)!: Uses wasm-pkg-client for loading dependencies

This is a fairly large PR because this dependency is used everywhere.
cargo component now uses the new wasm-pkg-tools toolchain to load deps,
which means that both OCI and Warg are supported. This tries to stay as
close to the original code style as possible, but I did have to make
large changes to how dependencies were being resolved to account for the
new library. There are a couple major breaking changes to be aware of and
one question to answer. This PR is already quite large, so I figured it
would be better to probably merge this and then deal with follow ups to
any of the outstanding questions below:

- Bindings are now generated every time. Where the actual dependency is
  being loaded from is now abstracted away by the client, so we can't
  check if those files have changed on disk. I personally don't think this
  is the worst thing, but if people really want that functionality, we
  can add support for that into the package tools.
- Offline deps currently does not work, but that is something that will
  be added into wasm-pkg-client instead of doing checks within cargo
  component itself.
- Currently the registries specified within Cargo.toml are not used in
  favor of using the config file for wasm-pkg-tools. I am not sure if we
  wanted to use that to override the wasm-pkg-tools config or just drop
  it entirely. I would like to address this one before merge.
- Until wasm-pkg-client has support for publishing, I have left support
  in to use warg directly. Once we have that added to wasm-pkg-tools, we
  can change the behavior here

Signed-off-by: Taylor Thomas <taylor@cosmonic.com>

Author: thomastaylor312

Cargo.lock

@@ -18,95 +18,102 @@ keywords = ["webassembly", "wasm", "components", "component-model"]
 repository = "https://github.com/bytecodealliance/cargo-component"
 
 [dependencies]
-cargo-component-core = { workspace = true }
 anyhow = { workspace = true }
+bytes = { workspace = true }
+cargo_metadata = { workspace = true }
+cargo-component-core = { workspace = true }
+cargo-config2 = { workspace = true }
 clap = { workspace = true }
-toml_edit = { workspace = true }
-pretty_env_logger = { workspace = true }
-log = { workspace = true }
-tokio = { workspace = true }
-tokio-util = { workspace = true }
+futures = { workspace = true }
 heck = { workspace = true }
-semver = { workspace = true }
-serde = { workspace = true }
-serde_json = { workspace = true }
 indexmap = { workspace = true }
-url = { workspace = true }
-wit-bindgen-rust = { workspace = true }
-wit-bindgen-core = { workspace = true }
-wit-parser = { workspace = true }
-wit-component = { workspace = true }
-wasm-metadata = { workspace = true }
-wasmparser = { workspace = true }
-parse_arg = { workspace = true }
-cargo_metadata = { workspace = true }
-cargo-config2 = { workspace = true }
 libc = { workspace = true }
-warg-protocol = { workspace = true }
-warg-crypto = { workspace = true }
-warg-client = { workspace = true }
+log = { workspace = true }
 p256 = { workspace = true }
+parse_arg = { workspace = true }
+pretty_env_logger = { workspace = true }
 rand_core = { workspace = true }
 rpassword = { workspace = true }
-futures = { workspace = true }
-bytes = { workspace = true }
-which = { workspace = true }
+semver = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
 shell-escape = "0.1.5"
 tempfile = { workspace = true }
+tokio = { workspace = true }
+tokio-util = { workspace = true }
+toml_edit = { workspace = true }
+url = { workspace = true }
+warg-client = { workspace = true }
+warg-crypto = { workspace = true }
+warg-protocol = { workspace = true }
+wasm-metadata = { workspace = true }
+wasm-pkg-client = { workspace = true }
+wasmparser = { workspace = true }
+which = { workspace = true }
+wit-bindgen-core = { workspace = true }
+wit-bindgen-rust = { workspace = true }
+wit-component = { workspace = true }
+wit-parser = { workspace = true }
 
 [dev-dependencies]
 assert_cmd = { workspace = true }
 predicates = { workspace = true }
-wat = { workspace = true }
-warg-server = { workspace = true }
 tempfile = { workspace = true }
+warg-server = { workspace = true }
 wasmprinter = { workspace = true }
+wat = { workspace = true }
 
 [workspace]
 members = ["crates/core", "crates/wit"]
 
 [workspace.dependencies]
-cargo-component-core = { path = "crates/core", version = "0.15.0-dev" }
-warg-protocol = "0.7.0"
-warg-crypto = "0.7.0"
-warg-client = "0.7.0"
-warg-server = "0.7.0"
 anyhow = "1.0.82"
-clap = { version = "4.5.4", features = ["derive"] }
-toml_edit = { version = "0.22.9", features = ["serde"] }
-pretty_env_logger = "0.5.0"
-log = "0.4.21"
-tokio = { version = "1.37.0", default-features = false, features = ["macros", "rt-multi-thread"] }
-tokio-util = "0.7.10"
-heck = "0.5.0"
-semver = "1.0.22"
-serde = { version = "1.0.197", features = ["derive"] }
-serde_json = "1.0.115"
-indexmap = "2.2.6"
-url = { version = "2.5.0", features = ["serde"] }
-wit-parser = "0.208.1"
-wit-component = "0.208.1"
-wasm-metadata = "0.208.1"
-parse_arg = "0.1.4"
+assert_cmd = "2.0.14"
+bytes = "1.6.0"
 cargo_metadata = "0.18.1"
+cargo-component-core = { path = "crates/core", version = "0.15.0-dev" }
 cargo-config2 = "0.1.24"
+clap = { version = "4.5.4", features = ["derive", "env"] }
+dirs = "5"
+futures = "0.3.30"
+heck = "0.5.0"
+indexmap = "2.2.6"
 libc = "0.2.153"
+log = "0.4.21"
+oci-distribution = "0.11"
 owo-colors = "4.0.0"
-unicode-width = "0.1.11"
 p256 = "0.13.2"
+parse_arg = "0.1.4"
+predicates = "3.1.0"
+pretty_env_logger = "0.5.0"
 rand_core = "0.6.4"
 rpassword = "7.3.1"
-futures = "0.3.30"
-bytes = "1.6.0"
-which = "6.0.1"
-wit-bindgen-rust = "0.25.0"
-wit-bindgen-core = "0.25.0"
+semver = "1"
+serde = { version = "1.0.197", features = ["derive"] }
+serde_json = "1.0.115"
 tempfile = "3.10.1"
-assert_cmd = "2.0.14"
-predicates = "3.1.0"
+tokio = { version = "1.37.0", default-features = false, features = [
+    "macros",
+    "rt-multi-thread",
+] }
+tokio-util = "0.7.10"
+toml_edit = { version = "0.22.9", features = ["serde"] }
+unicode-width = "0.1.11"
+url = { version = "2.5.0", features = ["serde"] }
+warg-client = "0.7.0"
+warg-crypto = "0.7.0"
+warg-protocol = "0.7.0"
+warg-server = "0.7.0"
+wasm-metadata = "0.208.1"
+wasm-pkg-client = { git = "https://github.com/thomastaylor312/wasm-pkg-tools.git", branch = "fix/cached_versions" }
 wasmparser = "0.208.1"
-wat = "1.208.1"
 wasmprinter = "0.208.1"
+wat = "1.208.1"
+which = "6.0.1"
+wit-bindgen-core = "0.25.0"
+wit-bindgen-rust = "0.25.0"
+wit-component = "0.208.1"
+wit-parser = "0.208.1"
 
 [profile.release]
 panic = "abort"

Cargo.toml

@@ -11,23 +11,26 @@ repository = { workspace = true }
 
 [dependencies]
 anyhow = { workspace = true }
+clap = { workspace = true }
+dirs = { workspace = true }
+futures = { workspace = true }
+indexmap = { workspace = true }
 libc = { workspace = true }
+log = { workspace = true }
 owo-colors = { workspace = true }
-unicode-width = { workspace = true }
-warg-crypto = { workspace = true }
-warg-protocol = { workspace = true }
-warg-client = { workspace = true }
-toml_edit = { workspace = true }
 semver = { workspace = true }
 serde = { workspace = true }
-indexmap = { workspace = true }
-futures = { workspace = true }
+tokio = { workspace = true }
+tokio-util = { workspace = true, features = ["io"] }
+toml_edit = { workspace = true }
+unicode-width = { workspace = true }
 url = { workspace = true }
+warg-client = { workspace = true }
+warg-crypto = { workspace = true }
+warg-protocol = { workspace = true }
+wasm-pkg-client = { workspace = true }
 wit-component = { workspace = true }
 wit-parser = { workspace = true }
-log = { workspace = true }
-tokio = { workspace = true }
-clap = { workspace = true }
 
 [target.'cfg(windows)'.dependencies.windows-sys]
 version = "0.52"

crates/core/Cargo.toml

@@ -1,8 +1,15 @@
 //! Module for common command implementation.
+use std::path::PathBuf;
 
-use crate::terminal::{Color, Terminal, Verbosity};
 use clap::{ArgAction, Args};
 
+use crate::terminal::{Color, Terminal, Verbosity};
+
+/// The environment variable name for setting a cache directory location
+pub const CACHE_DIR_ENV_VAR: &str = "CARGO_COMPONENT_CACHE_DIR";
+/// The environment variable name for setting a path to a config file
+pub const CONFIG_FILE_ENV_VAR: &str = "CARGO_COMPONENT_CONFIG_FILE";
+
 /// Common options for commands.
 #[derive(Args)]
 #[command(
@@ -24,6 +31,14 @@ pub struct CommonOptions {
     /// Coloring: auto, always, never
     #[clap(long = "color", value_name = "WHEN")]
     pub color: Option<Color>,
+
+    /// The path to the cache directory to store component dependencies.
+    #[clap(long = "cache-dir", env = CACHE_DIR_ENV_VAR)]
+    pub cache_dir: Option<PathBuf>,
+
+    /// The path to the pkg-tools config file
+    #[clap(long = "config", env = CONFIG_FILE_ENV_VAR)]
+    pub config: Option<PathBuf>,
 }
 
 impl CommonOptions {

crates/core/src/command.rs

@@ -2,22 +2,44 @@
 
 #![deny(missing_docs)]
 
+use std::path::PathBuf;
+use std::str::FromStr;
+
 use anyhow::Context;
 use semver::VersionReq;
-use std::str::FromStr;
-use warg_protocol::registry::PackageName;
+use wasm_pkg_client::PackageRef;
 
 pub mod command;
 pub mod lock;
 pub mod progress;
 pub mod registry;
 pub mod terminal;
 
+/// The root directory name used for default cargo component directories
+pub const CARGO_COMPONENT_DIR: &str = "cargo-component";
+/// The cache directory name used by default
+pub const CACHE_DIR: &str = "cache";
+
+/// Returns the path to the default cache directory, returning an error if a cache directory cannot be found.
+pub fn default_cache_dir() -> anyhow::Result<PathBuf> {
+    dirs::cache_dir()
+        .map(|p| p.join(CARGO_COMPONENT_DIR).join(CACHE_DIR))
+        .ok_or_else(|| anyhow::anyhow!("failed to find cache directory"))
+}
+
+/// A helper that fetches the default directory if the given directory is `None`.
+pub fn cache_dir(dir: Option<PathBuf>) -> anyhow::Result<PathBuf> {
+    match dir {
+        Some(dir) => Ok(dir),
+        None => default_cache_dir(),
+    }
+}
+
 /// Represents a versioned component package name.
 #[derive(Clone)]
 pub struct VersionedPackageName {
     /// The package name.
-    pub name: PackageName,
+    pub name: PackageRef,
     /// The optional package version.
     pub version: Option<VersionReq>,
 }

crates/core/src/lib.rs

@@ -10,8 +10,7 @@ use std::{
     path::{Path, PathBuf},
 };
 use toml_edit::{DocumentMut, Item, Value};
-use warg_crypto::hash::AnyHash;
-use warg_protocol::registry::PackageName;
+use wasm_pkg_client::{ContentDigest, PackageRef};
 
 /// The file format version of the lock file.
 const LOCK_FILE_VERSION: i64 = 1;
@@ -21,7 +20,7 @@ const LOCK_FILE_VERSION: i64 = 1;
 #[serde(rename_all = "kebab-case")]
 pub struct LockedPackage {
     /// The name of the locked package.
-    pub name: PackageName,
+    pub name: PackageRef,
     /// The registry the package was resolved from.
     ///
     /// Defaults to the default registry if not specified.
@@ -37,9 +36,10 @@ pub struct LockedPackage {
 
 impl LockedPackage {
     /// Gets the key used in sorting and searching the package list.
-    pub fn key(&self) -> (&PackageName, &str) {
+    pub fn key(&self) -> (&str, &str, &str) {
         (
-            &self.name,
+            self.name.namespace().as_ref(),
+            self.name.name().as_ref(),
             self.registry.as_deref().unwrap_or(DEFAULT_REGISTRY_NAME),
         )
     }
@@ -53,7 +53,7 @@ pub struct LockedPackageVersion {
     /// The version the package is locked to.
     pub version: Version,
     /// The digest of the package contents.
-    pub digest: AnyHash,
+    pub digest: ContentDigest,
 }
 
 impl LockedPackageVersion {
@@ -81,13 +81,20 @@ impl<'a> LockFileResolver<'a> {
     pub fn resolve(
         &'a self,
         registry: &str,
-        name: &PackageName,
+        package_ref: &PackageRef,
         requirement: &VersionReq,
     ) -> Result<Option<&'a LockedPackageVersion>> {
         if let Some(pkg) = self
             .0
             .packages
-            .binary_search_by_key(&(name, registry), LockedPackage::key)
+            .binary_search_by_key(
+                &(
+                    package_ref.namespace().as_ref(),
+                    package_ref.name().as_ref(),
+                    registry,
+                ),
+                LockedPackage::key,
+            )
             .ok()
             .map(|i| &self.0.packages[i])
         {
@@ -96,12 +103,12 @@ impl<'a> LockFileResolver<'a> {
                 .binary_search_by_key(&requirement.to_string().as_str(), LockedPackageVersion::key)
             {
                 let locked = &pkg.versions[index];
-                log::info!("dependency package `{name}` from registry `{registry}` with requirement `{requirement}` was resolved by the lock file to version {version}", version = locked.version);
+                log::info!("dependency package `{package_ref}` from registry `{registry}` with requirement `{requirement}` was resolved by the lock file to version {version}", version = locked.version);
                 return Ok(Some(locked));
             }
         }
 
-        log::info!("dependency package `{name}` from registry `{registry}` with requirement `{requirement}` was not in the lock file");
+        log::info!("dependency package `{package_ref}` from registry `{registry}` with requirement `{requirement}` was not in the lock file");
         Ok(None)
     }
 }