New feature: Allow authentication by personal tokens (#53)

* Correct usage of cargo-get in .gitlab-ci snippet

* Add note reg. incompatibility of cargo and thrussh

* README: Add .ssh/config for personal token usage

* README: Newer cargo versions do not allow URL-embedded passwords

* Refactor: Add function to build client with token

* Allow authentication with personal token

* Keep personal token in User

* Optionally use personal token for reading packages

* Optionally use personal token for DL urls

* README: Add usage of personal token authentication

* Consider only packages with type generic

---------

Co-authored-by: Morian Sonnet <morian.sonnet@isea.rwth-aachen.de>

Author: momoson

README.md

@@ -6,19 +6,28 @@ Say goodbye to your Git dependencies, `gitlab-cargo-shim` is a stateless SSH ser
 
 Access controls work like they do in GitLab, builds are scoped to users - if they don't have permission to the dependency they can't build it, it's that simple.
 
-Users are identified by their SSH keys from GitLab when connecting to the server and an [impersonation token][imp-token] will be generated for that run in order to pull available versions. Builds will insert their token as a username to the SSH server and the shim will use that to call the GitLab API.
+Users are either identified by their SSH keys from GitLab when connecting to the server or by an Gitlab personal-token. If no token is given, an [impersonation token][imp-token] will be generated for that run in order to pull available versions. Doing so requires ad admin personal token.
 
-To publish run `cargo package` and push the resulting `.crate` file to the GitLab package repository with a semver-compatible version string, to consume the package configure your `.cargo/config.toml` and `Cargo.toml` accordingly.
+To publish run `cargo package` and push the resulting `.crate` file to the GitLab package repository with a semver-compatible version string, to consume the package configure your `.cargo/config.toml`, `Cargo.toml` and, optionally, `.ssh/config` accordingly.
+
+At time of writing, `libssh2`, which `cargo` implicitly uses for communicating with the registry by SSH, is incompatible with rust's `thrussh`, due to non-overlapping ciphers. Hence, activating `net.git-fetch-with-cli` is necessary.
 
 ```toml
 # .cargo/config.toml
 [registries]
-my-gitlab-project = { index = "ssh://gitlab-cargo-shim.local/my-gitlab-group/my-gitlab-project" }
+my-gitlab-project = { index = "ssh://gitlab-cargo-shim.local/my-gitlab-group/my-gitlab-project/" }
+[net]
+git-fetch-with-cli = true
 
 # Cargo.toml
 [dependencies]
 my-crate = { version = "0.1", registry = "my-gitlab-project" }
 ```
+```ssh-config
+# .ssh/config (only if authentication by personal token is requires)
+Host gitlab-cargo-shim.local
+    User personal-token:<your-personal-token>
+```
 
 In your CI build, setup a `before_script` step to replace the connection string with one containing the CI token:
 
@@ -34,13 +43,13 @@ To release your package from CI, add a new pipeline step:
 
 ```yaml
 release-crate:
-  image: rust:1.62
+  image: rust:latest
   stage: deploy
   only: # release when a tag is pushed
     - tags
   before_script:
     - cargo install cargo-get
-    - export CRATE_NAME=$(cargo get --name) CRATE_VERSION=$(cargo get version)
+    - export CRATE_NAME=$(cargo-get package.name) CRATE_VERSION=$(cargo-get package.version)
     - export CRATE_FILE=${CRATE_NAME}-${CRATE_VERSION}.crate
   script:
     - cargo package
@@ -54,4 +63,4 @@ It's that easy. Go forth and enjoy your newfound quality of life improvements, R
 [gitlab-package-registry]: https://docs.gitlab.com/ee/user/packages/package_registry/index.html
 [imp-token]: https://docs.gitlab.com/ee/api/index.html#impersonation-tokens
 [envvar]: https://doc.rust-lang.org/cargo/reference/registries.html#using-an-alternate-registry
-[example-configuration]: https://github.com/w4/gitlab-cargo-shim/blob/main/config.toml
+[example-configuration]: https://github.com/w4/gitlab-cargo-shim/blob/main/config.toml

src/main.rs

@@ -247,7 +247,7 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {
 
         // fetch metadata from the provider
         let metadata = Arc::clone(&self.gitlab)
-            .fetch_metadata_for_release(path, crate_version)
+            .fetch_metadata_for_release(path, crate_version, self.user()?)
             .await?;
 
         // transform the `cargo metadata` output to the cargo index
@@ -291,7 +291,10 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {
 
         // fetch the impersonation token for the user we'll embed
         // the `dl` string.
-        let token = self.gitlab.fetch_token_for_user(self.user()?).await?;
+        let token = match &self.user()?.token {
+            None => self.gitlab.fetch_token_for_user(self.user()?).await?,
+            Some(token) => token.clone(),
+        };
 
         // generate the config for the user, containing the download
         // url template from gitlab and the impersonation token embedded
@@ -408,7 +411,7 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> thrussh::server:
                     info!(
                         "Successfully authenticated for GitLab user `{}` by {}",
                         &user.username,
-                        if by_ssh_key { "SSH Key" } else { "Build Token" },
+                        if by_ssh_key { "SSH Key" } else { "Build or Personal Token" },
                     );
                     self.user = Some(Arc::new(user));
                     self.finished_auth(Auth::Accept).await

src/providers/gitlab.rs

@@ -11,6 +11,7 @@ use std::{borrow::Cow, sync::Arc};
 use time::{Duration, OffsetDateTime};
 use tracing::{info_span, instrument, Instrument};
 use url::Url;
+use std::str::FromStr;
 
 pub struct Gitlab {
     client: reqwest::Client,
@@ -46,6 +47,19 @@ impl Gitlab {
             ssl_cert,
         })
     }
+
+    pub fn build_client_with_token(&self, token_field: &str, token: &str) -> anyhow::Result<reqwest::Client> {
+        let mut headers = header::HeaderMap::new();
+        headers.insert(
+            header::HeaderName::from_str(token_field)?,
+            header::HeaderValue::from_str(token)?,
+        );
+        let mut client_builder = reqwest::ClientBuilder::new().default_headers(headers);
+        if let Some(cert) = &self.ssl_cert {
+            client_builder = client_builder.add_root_certificate(cert.clone());
+        }
+        Ok(client_builder.build()?)
+    }
 }
 
 #[async_trait]
@@ -60,29 +74,43 @@ impl super::UserProvider for Gitlab {
             return Ok(None);
         };
 
-        if username == "gitlab-ci-token" {
+        if username == "gitlab-ci-token" || username == "personal-token" {
             // we're purposely not using `self.client` here as we don't
             // want to use our admin token for this request but still want to use any ssl cert provided.
-            let mut client_builder = reqwest::Client::builder();
-            if let Some(cert) = &self.ssl_cert {
-                client_builder = client_builder.add_root_certificate(cert.clone());
+            let client = self.build_client_with_token(if username == "gitlab-ci-token" { "JOB-TOKEN" } else { "PRIVATE-TOKEN" }, password);
+            if username == "gitlab-ci-token" {
+                let res: GitlabJobResponse = handle_error(
+                    client?
+                        .get(self.base_url.join("job/")?)
+                        .send()
+                        .await?,
+                )
+                .await?
+                .json()
+                .await?;
+
+                Ok(Some(User {
+                        id: res.user.id,
+                        username: res.user.username,
+                        ..Default::default()
+                }))
+            } else {
+                let res: GitlabUserResponse = handle_error(
+                    client?
+                        .get(self.base_url.join("user/")?)
+                        .send()
+                        .await?,
+                )
+                .await?
+                .json()
+                .await?;
+
+                Ok(Some(User {
+                        id: res.id,
+                        username: res.username,
+                        token: Some(password.to_string()),
+                }))
             }
-            let client = client_builder.build();
-            let res: GitlabJobResponse = handle_error(
-                client?
-                    .get(self.base_url.join("job/")?)
-                    .header("JOB-TOKEN", password)
-                    .send()
-                    .await?,
-            )
-            .await?
-            .json()
-            .await?;
-
-            Ok(Some(User {
-                id: res.user.id,
-                username: res.user.username,
-            }))
         } else {
             Ok(None)
         }
@@ -101,6 +129,7 @@ impl super::UserProvider for Gitlab {
         Ok(res.user.map(|u| User {
             id: u.id,
             username: u.username,
+            ..Default::default()
         }))
     }
 
@@ -149,15 +178,23 @@ impl super::PackageProvider for Gitlab {
                 query.append_pair("per_page", itoa::Buffer::new().format(100u16));
                 query.append_pair("pagination", "keyset");
                 query.append_pair("sort", "asc");
-                query.append_pair("sudo", itoa::Buffer::new().format(do_as.id));
+                if do_as.token.is_none() {
+                    query.append_pair("sudo", itoa::Buffer::new().format(do_as.id));
+                }
             }
             uri
         });
 
         let futures = FuturesUnordered::new();
 
+        let client = match &do_as.token {
+            None => self.client.clone(),
+            Some(token) => self.build_client_with_token("PRIVATE-TOKEN", token)?
+        };
+        let client = Arc::new(client);
+
         while let Some(uri) = next_uri.take() {
-            let res = handle_error(self.client.get(uri).send().await?).await?;
+            let res = handle_error(client.get(uri).send().await?).await?;
 
             if let Some(link_header) = res.headers().get(header::LINK) {
                 let mut link_header = parse_link_header::parse_with_rel(link_header.to_str()?)?;
@@ -167,10 +204,15 @@ impl super::PackageProvider for Gitlab {
                 }
             }
 
-            let res: Vec<GitlabPackageResponse> = res.json().await?;
+            let res: Vec<GitlabPackageResponse> = res.json::<Vec<GitlabPackageResponse>>()
+                                                     .await?
+                                                     .into_iter()
+                                                     .filter(|release| release.package_type == "generic")
+                                                     .collect();
 
             for release in res {
                 let this = Arc::clone(&self);
+                let client = Arc::clone(&client);
 
                 futures.push(tokio::spawn(
                     async move {
@@ -189,7 +231,7 @@ impl super::PackageProvider for Gitlab {
                         });
 
                         let package_files: Vec<GitlabPackageFilesResponse> = handle_error(
-                            this.client
+                            client
                                 .get(format!(
                                     "{}/projects/{}/packages/{}/package_files",
                                     this.base_url,
@@ -239,10 +281,15 @@ impl super::PackageProvider for Gitlab {
         &self,
         path: &Self::CratePath,
         version: &str,
+        do_as: &User,
     ) -> anyhow::Result<cargo_metadata::Metadata> {
         let uri = self.base_url.join(&path.metadata_uri(version))?;
+        let client = match &do_as.token {
+            None => self.client.clone(),
+            Some(token) => self.build_client_with_token("PRIVATE-TOKEN", token)?
+        };
 
-        Ok(handle_error(self.client.get(uri).send().await?)
+        Ok(handle_error(client.get(uri).send().await?)
             .await?
             .json()
             .await?)
@@ -315,6 +362,7 @@ pub struct GitlabPackageResponse {
     pub id: u64,
     pub name: String,
     pub version: String,
+    pub package_type: String,
     #[serde(rename = "_links")]
     pub links: GitlabPackageLinksResponse,
 }

src/providers/mod.rs

@@ -31,15 +31,17 @@ pub trait PackageProvider {
         &self,
         path: &Self::CratePath,
         version: &str,
+        do_as: &User,
     ) -> anyhow::Result<cargo_metadata::Metadata>;
 
     fn cargo_dl_uri(&self, project: &str, token: &str) -> anyhow::Result<String>;
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Default)]
 pub struct User {
     pub id: u64,
     pub username: String,
+    pub token: Option<String>,
 }
 
 pub type ReleaseName = Arc<str>;