diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..228c104 --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +# Copy this file as your .env +POSTGRES_DB=vinted-rs +POSTGRES_PASSWORD=postgres +POSTGRES_USER=postgres \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index 81bcc8d..f607180 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,4 +2,4 @@ static/*.html linguist-generated # Mark all HTML files in scrapping/vinted-db-feeder/data/raw/ as generated -scrapping/vinted-db-feeder/data/raw/*.html linguist-generated +scrapping/vinted-db-feeder/data/raw/**/*.html linguist-generated diff --git a/.gitignore b/.gitignore index 53b9c8c..4f182f6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ Cargo.lock /**/results/ docker/query.sh -src/tests/output \ No newline at end of file +src/tests/output +.env \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 382a9fe..6b35051 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,24 @@ # Changelog -# 0.9.2 (WIP) +# 0.10.0 (WIP) ## Fixed -[#115](https://github.com/ThalosES/vinted-rs/pull/115) : Rexported Redis crate +- Re-exported Redis crate [#115](https://github.com/ThalosES/vinted-rs/pull/115) + +- Added support for fields that may come as boolean or integer [#118](https://github.com/ThalosES/vinted-rs/pull/118) + +## Improved + +- Removed hardcoded DB strings and introduced a `.env` file required for feature `Advanced Items` [#117](https://github.com/ThalosES/vinted-rs/pull/117) # 0.9.1 (2024-09-24) [#111](https://github.com/TuTarea/vinted-rs/pull/111/) -⚠️Not using `develop` branch any more. Leaving main as "canary"/develop channel and using releases to declare _stable_ and _experimental_ versions + +⚠️Not using [`develop`](https://github.com/ThalosES/vinted-rs/tree/develop) branch any more. Leaving main as "canary"/develop channel and using releases to declare _stable_ and _experimental_ versions ## Fixed -#108 : Updated Redis to 0.27.2 -#105 : Updated TypeBuilder to 0.20 -#109 : Solved vulnerable dependency for Db Feeder +- Updated Redis to 0.27.2 [#108](https://github.com/ThalosES/vinted-rs/pull/108) +- Updated TypeBuilder to 0.20 [#105](https://github.com/ThalosES/vinted-rs/pull/105) +- Solved vulnerable dependency for Db Feeder [#109](https://github.com/ThalosES/vinted-rs/pull/109) # 0.9.0 (2024-07-28) [#97](https://github.com/TuTarea/vinted-rs/pull/97/) diff --git a/Cargo.toml b/Cargo.toml index 144ec1e..3447241 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,9 @@ +[workspace] +members = ["examples/filter_example", "."] + [package] name = "vinted-rs" -version = "0.9.2" +version = "0.10.0" edition = "2021" repository = "https://github.com/TuTarea/vinted-rs" authors = [ @@ -34,14 +37,16 @@ reqwest_cookie_store = "0.8" typed-builder = "0.20" fang = { version = "0.10.3", features = ["asynk"], default-features = false } redis-macros = { version = "0.4.2", optional = true } -redis = { version = "0.27.2", optional = true, features = ["tokio-comp", "aio"] } +redis = { version = "0.27.5", optional = true, features = [ + "tokio-comp", + "aio", +] } serde_json = { version = "1.0.91" } log = "0.4.20" lazy_static = "1.4.0" +dotenvy = "0.15" [dev-dependencies] env_logger = "0.11.5" -redis-macros = { version = "0.4.2" } -redis = { version = "0.27.2" } [dependencies.bb8-postgres] diff --git a/Makefile b/Makefile index 63a0306..fdc40f5 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,20 @@ +include .env + db: docker run --rm -d --name postgres -p 5432:5432 \ - -e POSTGRES_DB=vinted-rs \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=$(POSTGRES_DB) \ + -e POSTGRES_USER=$(POSTGRES_USER) \ + -e POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) \ postgres:latest diesel: - DATABASE_URL=postgres://postgres:postgres@localhost/vinted-rs diesel migration run + DATABASE_URL=postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@localhost/$(POSTGRES_DB) diesel migration run stop: docker kill postgres clippy: - cargo clippy --all-features \ No newline at end of file + cargo clippy --all-features + +env: + \ No newline at end of file diff --git a/README.md b/README.md index 31f6453..e9137d5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# Vinted-rs: A Vinted API wrapper +
+ +# Vinted-rs: A Vinted API wrapper in Rust [![github]](https://github.com/TuTarea/vinted-rs/) [![crates-io]](https://crates.io/crates/vinted-rs) [![docs-rs]](https://docs.rs/vinted-rs/latest/vinted_rs/) @@ -6,17 +8,7 @@ [crates-io]: https://img.shields.io/badge/crates.io-fc8d62?style=for-the-badge&labelColor=555555&logo=rust [docs-rs]: https://img.shields.io/badge/docs.rs-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs -## Table of Contents - -- [Vinted-rs: A Vinted API wrapper](#vinted-rs-a-vinted-api-wrapper) - - [Table of Contents](#table-of-contents) - - [Installation](#installation) - - [DB setup](#db-setup) - - [Create a migration](#create-a-migration) - - [Run a Docker container with PostgreSQL](#run-a-docker-container-with-postgresql) - - [Run migrations](#run-migrations) - - [Stop DB](#stop-db) - - [Running Tests](#running-tests) +
## Installation @@ -24,73 +16,102 @@ Via `cargo` you can add the library to your project's `Cargo.toml` ```toml [dependencies] -vinted-rs = "0.9.1" +vinted-rs = { version = "0.10.0", + #features = ["advanced_filters", "redis"] + } ``` -## DB setup +## Features + +| Feature | Description | Example | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------- | +| [Advanced Filters](#advanced-filters) | Uses the data pulled by the [scrapping module](./scrapping/vinted-db-feeder/), which is stored in the diesel [migrations](./migrations/) folder. | [✅](./examples/filter_example/) | +| [Redis](#redis) | Allows recovered results to be cached using a Redis instance | ❌ | + +### Advanced filters + +> This feature requires [setting up a Postgres Database](#database-set-up) + +Uses the data pulled by the [scrapping module](./scrapping/vinted-db-feeder/), which is stored in the diesel [migrations](./migrations/) folder. + +#### Environment set-up +1. Copy the `.env.example` + + ```sh + cp .env.example .env + ``` + +2. Modify the variables to your liking + +#### Database set-up Advanced filtering features must require this setup before running. -- First start installing diesel-cli (in order to run the migrations in PostgreSQL database) +1. ⚠️ `diesel-cli` installation may fail if you do not have `libpq` library installed. To install `libpq`, just install PostgreSQL package on your machine. -⚠️**Very important:** diesel-cli installation may fail if you do not have `libpq` library installed. + - In `Arch` based is only necessary to install this package. -To install `libpq`, just install PostgreSQL package on your machine. + ```bash + sudo pacman -S postgresql-libs + ``` -In `Arch` based is only necessary to install this package. + - In `Debian` based distributions is only necessary to install this package. -```bash -sudo pacman -S postgresql-libs -``` + ```bash + sudo apt install libpq-dev + ``` -In `Debian` based distributions is only necessary to install this package. +2. Install `diesel-cli` in order to run the migrations in PostgreSQL database + + ```bash + cargo install diesel_cli --features=postgres --no-default-features + ``` -```bash -sudo apt install libpq-dev -``` +**Available interactions** (See [Makefile](./Makefile)) -```bash -cargo install diesel_cli --features=postgres --no-default-features -``` +1. Create a migration -### Create a migration + ```bash + mkdir -p migrations # + diesel migration generate my_migration + ``` -```bash -mkdir migrations -``` + Program after that `up.sql` and `down.sql` scripts. -```bash -diesel migration generate my_migration -``` +2. Run a Docker container with PostgreSQL -Program after that `up.sql` and `down.sql` scripts. + - See in [Makefile](https://github.com/ThalosES/vinted-rs/blob/main/Makefile) -### Run a Docker container with PostgreSQL + ```bash + make db + ``` -- See in [Makefile](https://github.com/TuTarea/vinted-rs/blob/main/Makefile) +3. Run migrations -```bash -make db -``` + ```bash + make diesel + ``` -### Run migrations +4. Stop DB -```bash -make diesel -``` + ```bash + make stop + ``` + +#### Testing set-up -### Stop DB +> This step requires completing the [DB setup](#database-set-up) ```bash -make stop +cargo test ``` -## Running Tests +### Redis -⚠️**Very important:** Before running tests is important to do the [DB setup](#db-setup) +This feature allows recovered results to be cached using a Redis instance. -Then run the tests +A development instance can be created using: ```bash -cargo test -``` +make cache +``` \ No newline at end of file diff --git a/examples/filter_example/.gitignore b/examples/filter_example/.gitignore new file mode 100644 index 0000000..834e321 --- /dev/null +++ b/examples/filter_example/.gitignore @@ -0,0 +1,2 @@ +.venv +result/ \ No newline at end of file diff --git a/examples/filter_example/Cargo.toml b/examples/filter_example/Cargo.toml index 9e6379a..8d5123d 100644 --- a/examples/filter_example/Cargo.toml +++ b/examples/filter_example/Cargo.toml @@ -1,11 +1,17 @@ [package] name = "filter_example" -version = "0.1.0" +version = "0.10.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -vinted-rs = { path = "../../", features = ["advanced_filters"]} -bb8-postgres = {version = "0.8", features = ["with-serde_json-1" , "with-uuid-1" , "with-chrono-0_4"]} +vinted-rs = { path = "../../", features = ["advanced_filters"] } +bb8-postgres = { version = "0.8", features = [ + "with-serde_json-1", + "with-uuid-1", + "with-chrono-0_4", +] } tokio = { version = "1", features = ["full"] } +dotenvy = { version = "0.15.7" } +lazy_static = { version = "1.4.0" } diff --git a/examples/filter_example/README.md b/examples/filter_example/README.md new file mode 100644 index 0000000..ae2e72c --- /dev/null +++ b/examples/filter_example/README.md @@ -0,0 +1,20 @@ +# Advanced filter example project + +## Rust set-up + +Refer to the [main README *Install* section](../../README.md#installation) and its subsections + +## Python set-up + +1. Create a Virtual Environment + +```bash +python -m venv .venv +``` + +2. Install the requirements + +```bash +source .venv/bin/activate #linux +pip install -r requirements.txt +``` \ No newline at end of file diff --git a/examples/filter_example/src/main.rs b/examples/filter_example/src/main.rs index 06c56d8..97a349a 100644 --- a/examples/filter_example/src/main.rs +++ b/examples/filter_example/src/main.rs @@ -1,12 +1,23 @@ use bb8_postgres::tokio_postgres::NoTls; - +use lazy_static::lazy_static; use std::env; use vinted_rs::{db::DbController, queries::Host, Filter, VintedWrapper}; +lazy_static! { + pub static ref POSTGRES_DB: String = + std::env::var("POSTGRES_DB").unwrap_or(String::from("vinted-rs")); + pub static ref POSTGRES_USER: String = + std::env::var("POSTGRES_USER").unwrap_or(String::from("postgres")); + pub static ref POSTGRES_PASSWORD: String = + std::env::var("POSTGRES_PASSWORD").unwrap_or(String::from("postgres")); +} + #[tokio::main] async fn main() { let args: Vec = env::args().collect(); + let _ = dotenvy::dotenv(); //Load at runtime + if args.len() < 2 { println!("Please provide the host as a command-line parameter."); return; @@ -15,9 +26,19 @@ async fn main() { let host_arg = args[1].as_str(); let host: Host = host_arg.into(); - let db = DbController::new("postgres://postgres:postgres@localhost/vinted-rs", 5, NoTls) + let vinted = VintedWrapper::new_with_host(host); + println!("Host: {}", vinted.get_host()); + + let db_uri = &format!( + "postgres://{}:{}@localhost/{}?sslmode=disable", + POSTGRES_USER.clone(), + POSTGRES_PASSWORD.clone(), + POSTGRES_DB.clone() + ); + + let db = DbController::new(&db_uri, 5, NoTls) .await - .unwrap(); + .expect("Broken connection to Database, please set it up correctly"); let adidas = db.get_brand_by_name(&"Adidas").await.unwrap(); let nike = db.get_brand_by_name(&"Nike").await.unwrap(); @@ -30,10 +51,6 @@ async fn main() { .price_to(Some(20.0)) .build(); - let vinted = VintedWrapper::new_with_host(host); - - println!("Host: {}", vinted.get_host()); - let items = vinted .get_items(&filter, 10, None, None, None) .await diff --git a/src/lib.rs b/src/lib.rs index 68e2831..c83be1b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,22 +29,25 @@ Cookie automatic authentication using CookieStore ```rust use bb8_postgres::tokio_postgres::NoTls; -use vinted_rs::{ - db::DbController, - model::{filter::brand::Brand, filter::Filter}, -}; - -use vinted_rs::VintedWrapper; - -const DB_URL: &str = "postgres://postgres:postgres@localhost/vinted-rs"; -const POOL_SIZE: u32 = 5; - - +use lazy_static::lazy_static; +use std::env; +use vinted_rs::{db::DbController, queries::Host, Filter, VintedWrapper}; + +lazy_static! { + pub static ref POSTGRES_DB: String = + std::env::var("POSTGRES_DB").unwrap_or(String::from("vinted-rs")); + pub static ref POSTGRES_USER: String = + std::env::var("POSTGRES_USER").unwrap_or(String::from("postgres")); + pub static ref POSTGRES_PASSWORD: String = + std::env::var("POSTGRES_PASSWORD").unwrap_or(String::from("postgres")); +} #[tokio::main] async fn main() { let args: Vec = env::args().collect(); + let _ = dotenvy::dotenv(); //Load at runtime + if args.len() < 2 { println!("Please provide the host as a command-line parameter."); return; @@ -53,9 +56,19 @@ async fn main() { let host_arg = args[1].as_str(); let host: Host = host_arg.into(); - let db = DbController::new("postgres://postgres:postgres@localhost/vinted-rs", 5, NoTls) + let vinted = VintedWrapper::new_with_host(host); + println!("Host: {}", vinted.get_host()); + + let db_uri = &format!( + "postgres://{}:{}@localhost/{}?sslmode=disable", + POSTGRES_USER.clone(), + POSTGRES_PASSWORD.clone(), + POSTGRES_DB.clone() + ); + + let db = DbController::new(&db_uri, 5, NoTls) .await - .unwrap(); + .expect("Broken connection to Database, please set it up correctly"); let adidas = db.get_brand_by_name(&"Adidas").await.unwrap(); let nike = db.get_brand_by_name(&"Nike").await.unwrap(); @@ -63,25 +76,30 @@ async fn main() { let brands = format!("{},{}", adidas.id, nike.id); let filter = Filter::builder() - .brand_ids(brands) - .price_from(15) - .price_to(20) + .brand_ids(Some(brands)) + .price_from(Some(15.0)) + .price_to(Some(20.0)) .build(); - let vinted = VintedWrapper::new_with_host(host); - - println!("Host: {}", vinted.get_host()); - - let items = vinted.get_items(&filter, 10).await.unwrap(); + let items = vinted + .get_items(&filter, 10, None, None, None) + .await + .unwrap(); if items.items.is_empty() { - println!("No items found"); + } else { + for item in items.items { + let advanced = vinted + .get_advanced_item(item.id, None, None, None) + .await + .unwrap(); + println!("{}", advanced); + } } - println!("{}", items); - } + ``` */ #[cfg(feature = "advanced_filters")] diff --git a/src/model.rs b/src/model.rs index 5e0307b..13b1fd0 100644 --- a/src/model.rs +++ b/src/model.rs @@ -67,6 +67,9 @@ pub mod payment_method; /// - `photo`: The photo of the user. pub mod user; +/// Serde configuration attributes to handle wrongly typed items +pub mod serde_config; + #[cfg(feature = "redis")] pub use redis_macros::{FromRedisValue, ToRedisArgs}; pub use serde::{Deserialize, Serialize}; diff --git a/src/model/item.rs b/src/model/item.rs index 36abf20..d691a66 100644 --- a/src/model/item.rs +++ b/src/model/item.rs @@ -1,4 +1,5 @@ use super::{photo::Photo, user::AdvancedUser}; +use crate::model::serde_config::bool_from_int_or_bool; use crate::model::{Deserialize, Serialize}; #[cfg(feature = "redis")] use crate::model::{FromRedisValue, ToRedisArgs}; @@ -17,7 +18,7 @@ pub struct Item { pub price: String, pub photo: Option, pub url: String, - pub is_visible: i32, + pub is_visible: bool, pub promoted: bool, pub favourite_count: i32, } @@ -131,49 +132,116 @@ pub struct AdvancedItem { pub user: AdvancedUser, // Some flags - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub is_for_sell: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub is_for_swap: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub is_for_give_away: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub is_handicraft: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub is_processing: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub is_draft: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub promoted: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub package_size_standard: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub related_catalogs_enabled: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub is_hidden: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub is_reserved: Option, - // More flags, just in i32 #[serde(skip_serializing_if = "Option::is_none")] pub reserved_for_user_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub is_visible: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] + pub is_visible: Option, + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub is_visible_new: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub is_unisex: Option, + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] + pub is_unisex: Option, - /* - WARN:Leaving is_closed commented, since its type [i32, bool] - is not stable and could cause issues - */ - // #[serde(skip_serializing_if = "Option::is_none")] - // pub is_closed: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] + pub is_closed: Option, + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub is_closed_new: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub is_delayed_publication: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "bool_from_int_or_bool", + default + )] pub can_be_sold: Option, } diff --git a/src/model/serde_config.rs b/src/model/serde_config.rs new file mode 100644 index 0000000..246f0dc --- /dev/null +++ b/src/model/serde_config.rs @@ -0,0 +1,31 @@ +use serde::{Deserialize, Deserializer}; +use serde_json::Value; + +pub fn bool_from_int_or_bool<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + match Value::deserialize(deserializer)? { + Value::Bool(b) => Ok(Some(b)), + Value::Number(n) => { + let Some(i) = n.as_i64() else { + return Err(serde::de::Error::custom( + "expected an integer for optional boolean field", + )); + }; + + match i { + 0 => Ok(Some(false)), + 1 => Ok(Some(true)), + _ => Err(serde::de::Error::custom( + "expected 0 or 1 for optional boolean field", + )), + } + } + + Value::Null => Ok(None), // Handle `null` as `None` + _ => Err(serde::de::Error::custom( + "expected a boolean, integer, or null for optional boolean field", + )), + } +} diff --git a/src/queries.rs b/src/queries.rs index 43f635f..e7bc662 100644 --- a/src/queries.rs +++ b/src/queries.rs @@ -32,6 +32,7 @@ use fang::FangError; use lazy_static::lazy_static; use log::debug; +use log::error; use rand::Rng; use reqwest::Client; use reqwest::Proxy; @@ -39,6 +40,7 @@ use reqwest::Response; use reqwest::StatusCode; use reqwest_cookie_store::CookieStore; use reqwest_cookie_store::CookieStoreMutex; +use std::fmt; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::sync::Arc; @@ -62,7 +64,9 @@ pub enum CookieError { #[derive(Error, Debug)] pub enum VintedWrapperError { #[error(transparent)] - SerdeError(#[from] serde_json::Error), + ReqWestError(#[from] reqwest::Error), + #[error(transparent)] + SerdeError(#[from] SerdeJSONError), #[error(transparent)] CookiesError(#[from] CookieError), #[error("Number of items must be non-zero value")] @@ -71,9 +75,28 @@ pub enum VintedWrapperError { ItemError(StatusCode, Option, String), } -impl From for VintedWrapperError { - fn from(value: reqwest::Error) -> Self { - VintedWrapperError::CookiesError(CookieError::ReqWestError(value)) +#[derive(Debug, Error)] +pub struct SerdeJSONError { + raw_json: String, + serde_error: serde_json::Error, +} + +impl SerdeJSONError { + fn new(raw_json: String, serde_error: serde_json::Error) -> Self { + SerdeJSONError { + raw_json, + serde_error, + } + } +} + +impl fmt::Display for SerdeJSONError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "JSON: {}, SerdeError: {}", + self.raw_json, self.serde_error + ) } } @@ -703,9 +726,20 @@ impl<'a> VintedWrapper<'a> { match json.status() { StatusCode::OK => { - let items: Items = json.json().await?; - Ok(items) + // First, get the response body as text to enable debugging if deserialization fails + let raw_json = json.text().await?; + + // Try to parse the JSON into Items + match serde_json::from_str::(&raw_json) { + Ok(items) => Ok(items), + Err(serde_error) => { + let error = SerdeJSONError::new(raw_json, serde_error); + error!("Failed to deserialize: {}", error); // Or use a logger + Err(VintedWrapperError::SerdeError(error)) + } + } } + code => { let retry_after = json .headers() @@ -758,17 +792,18 @@ impl<'a> VintedWrapper<'a> { match json.status() { StatusCode::OK => { - let response_text = json.text().await?; - let result: Result = - serde_json::from_str(&response_text); - - if result.is_err() { - log::error!("{}", &response_text) + let raw_json = json.text().await?; + match serde_json::from_str::(&raw_json) { + Ok(items) => Ok(items.item), + Err(serde_error) => { + // Log or debug the raw JSON content + let error = SerdeJSONError::new(raw_json, serde_error); + error!("Failed to deserialize: {}", error); // Or use a logger + Err(VintedWrapperError::SerdeError(error)) + } } - - let items = result?; - Ok(items.item) } + code => { let retry_after = json .headers() diff --git a/src/tests.rs b/src/tests.rs index 796b824..efc462a 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,3 +1,21 @@ +use lazy_static::lazy_static; + +lazy_static! { + pub static ref POSTGRES_DB: String = + std::env::var("POSTGRES_DB").unwrap_or(String::from("vinted-rs")); + pub static ref POSTGRES_USER: String = + std::env::var("POSTGRES_USER").unwrap_or(String::from("postgres")); + pub static ref POSTGRES_PASSWORD: String = + std::env::var("POSTGRES_PASSWORD").unwrap_or(String::from("postgres")); + pub static ref DB_URI: String = { + dotenvy::dotenv().ok(); // Load environment variables from .env file + format!( + "postgres://{}:{}@localhost/{}?sslmode=disable", + *POSTGRES_USER, *POSTGRES_PASSWORD, *POSTGRES_DB + ) + }; +} + #[cfg(test)] pub mod db; #[cfg(test)] diff --git a/src/tests/db.rs b/src/tests/db.rs index 4797293..9ccb7f8 100644 --- a/src/tests/db.rs +++ b/src/tests/db.rs @@ -1,15 +1,15 @@ use crate::{ db::DbController, model::filter::{brand::Brand, category::Category, country::Country, size::Size}, + tests::DB_URI, }; use bb8_postgres::tokio_postgres::NoTls; -const DB_URL: &str = "postgres://postgres:postgres@localhost/vinted-rs"; const POOL_SIZE: u32 = 5; #[tokio::test] async fn test_get_brand_by_name() { - let db: DbController = DbController::new(DB_URL, POOL_SIZE, NoTls).await.unwrap(); + let db: DbController = DbController::new(&DB_URI, POOL_SIZE, NoTls).await.unwrap(); let brand_name: String = String::from("adidas"); @@ -27,7 +27,7 @@ async fn test_get_brand_by_name() { #[tokio::test] async fn test_get_brands_by_name() { - let db: DbController = DbController::new(DB_URL, POOL_SIZE, NoTls).await.unwrap(); + let db: DbController = DbController::new(&DB_URI, POOL_SIZE, NoTls).await.unwrap(); let brand_name: String = String::from("adidas"); @@ -38,7 +38,7 @@ async fn test_get_brands_by_name() { #[tokio::test] async fn test_get_category_by_name() { - let db: DbController = DbController::new(DB_URL, POOL_SIZE, NoTls).await.unwrap(); + let db: DbController = DbController::new(&DB_URI, POOL_SIZE, NoTls).await.unwrap(); let category_name: String = String::from("Women"); @@ -59,7 +59,7 @@ async fn test_get_category_by_name() { #[tokio::test] async fn test_get_country_by_iso() { - let db: DbController = DbController::new(DB_URL, POOL_SIZE, NoTls).await.unwrap(); + let db: DbController = DbController::new(&DB_URI, POOL_SIZE, NoTls).await.unwrap(); let c = db.get_country_by_iso(&String::from("ES")).await.unwrap(); assert_eq!( @@ -76,7 +76,7 @@ async fn test_get_country_by_iso() { #[tokio::test] async fn test_get_size_by_title_and_type() { - let db: DbController = DbController::new(DB_URL, POOL_SIZE, NoTls).await.unwrap(); + let db: DbController = DbController::new(&DB_URI, POOL_SIZE, NoTls).await.unwrap(); let size = db .get_size_by_title_and_type( &String::from("ES"), @@ -102,7 +102,7 @@ async fn test_get_size_by_title_and_type() { } #[tokio::test] async fn test_get_sizes_for_category() { - let db: DbController = DbController::new(DB_URL, POOL_SIZE, NoTls).await.unwrap(); + let db: DbController = DbController::new(&DB_URI, POOL_SIZE, NoTls).await.unwrap(); let sizes = db.get_sizes_for_category(5).await.unwrap(); assert!(sizes.into_iter().all(|size| { size.category_id == 5 })); diff --git a/src/tests/queries.rs b/src/tests/queries.rs index 1ca5fc5..35b94ab 100644 --- a/src/tests/queries.rs +++ b/src/tests/queries.rs @@ -1,11 +1,11 @@ use crate::db::DbController; use crate::model::filter::{Currency, Filter}; use crate::queries::VintedWrapperError; +use crate::tests::DB_URI; use crate::VintedWrapper; use bb8_postgres::tokio_postgres::NoTls; use env_logger; -const DB_URL: &str = "postgres://postgres:postgres@localhost/vinted-rs"; const POOL_SIZE: u32 = 5; fn _calculate_color_props(hex_color1: &str) -> (f64, f64, f64) { @@ -41,19 +41,23 @@ async fn test_get_item_query_text() { Ok(items) => { assert!(items.items.len() <= 1); } - Err(err) => match err { - VintedWrapperError::ItemNumberError => unreachable!(), - VintedWrapperError::ItemError(_, _, _) => (), - VintedWrapperError::CookiesError(_) => (), - VintedWrapperError::SerdeError(_) => (), - }, + Err(err) => { + log::error!("{:#?}", err); + match err { + VintedWrapperError::ItemNumberError => unreachable!(), + VintedWrapperError::ItemError(_, _, _) => unreachable!(), + VintedWrapperError::CookiesError(_) => (), + VintedWrapperError::SerdeError(_) => unreachable!(), + VintedWrapperError::ReqWestError(_) => unreachable!(), + } + } }; } #[tokio::test] async fn test_get_item_brands() { let vinted = VintedWrapper::new(); - let db: DbController = DbController::new(DB_URL, POOL_SIZE, NoTls).await.unwrap(); + let db: DbController = DbController::new(&DB_URI, POOL_SIZE, NoTls).await.unwrap(); let brand = db.get_brand_by_name(&String::from("Adidas")).await.unwrap(); let filter: Filter = Filter::builder() @@ -69,19 +73,23 @@ async fn test_get_item_brands() { } assert_eq!(result.unwrap().brand_title, brand.title); } - Err(err) => match err { - VintedWrapperError::ItemNumberError => unreachable!(), - VintedWrapperError::ItemError(_, _, _) => (), - VintedWrapperError::CookiesError(_) => (), - VintedWrapperError::SerdeError(_) => (), - }, + Err(err) => { + log::error!("{:#?}", err); + match err { + VintedWrapperError::ItemNumberError => unreachable!(), + VintedWrapperError::ItemError(_, _, _) => unreachable!(), + VintedWrapperError::CookiesError(_) => (), + VintedWrapperError::SerdeError(_) => unreachable!(), + VintedWrapperError::ReqWestError(_) => unreachable!(), + } + } }; } #[tokio::test] async fn test_get_items_brands() { let vinted = VintedWrapper::new(); - let db: DbController = DbController::new(DB_URL, POOL_SIZE, NoTls).await.unwrap(); + let db: DbController = DbController::new(&DB_URI, POOL_SIZE, NoTls).await.unwrap(); let brand = db.get_brand_by_name(&String::from("Adidas")).await.unwrap(); let filter: Filter = Filter::builder() @@ -94,12 +102,16 @@ async fn test_get_items_brands() { assert_eq!(item.brand_title, brand.title); } } - Err(err) => match err { - VintedWrapperError::ItemNumberError => unreachable!(), - VintedWrapperError::ItemError(_, _, _) => (), - VintedWrapperError::SerdeError(_) => (), - VintedWrapperError::CookiesError(_) => (), - }, + Err(err) => { + log::error!("{:#?}", err); + match err { + VintedWrapperError::ItemNumberError => unreachable!(), + VintedWrapperError::ItemError(_, _, _) => unreachable!(), + VintedWrapperError::CookiesError(_) => (), + VintedWrapperError::SerdeError(_) => unreachable!(), + VintedWrapperError::ReqWestError(_) => unreachable!(), + } + } }; } @@ -130,12 +142,16 @@ async fn test_get_items_catalogs_no_db() { ); }); } - Err(err) => match err { - VintedWrapperError::ItemNumberError => unreachable!(), - VintedWrapperError::ItemError(_, _, _) => (), - VintedWrapperError::CookiesError(_) => (), - VintedWrapperError::SerdeError(_) => (), - }, + Err(err) => { + log::error!("{:#?}", err); + match err { + VintedWrapperError::ItemNumberError => unreachable!(), + VintedWrapperError::ItemError(_, _, _) => unreachable!(), + VintedWrapperError::CookiesError(_) => (), + VintedWrapperError::SerdeError(_) => unreachable!(), + VintedWrapperError::ReqWestError(_) => unreachable!(), + } + } }; } @@ -160,12 +176,16 @@ async fn test_get_items_by_price() { assert!(ok); } - Err(err) => match err { - VintedWrapperError::ItemNumberError => unreachable!(), - VintedWrapperError::ItemError(_, _, _) => (), - VintedWrapperError::CookiesError(_) => (), - VintedWrapperError::SerdeError(_) => (), - }, + Err(err) => { + log::error!("{:#?}", err); + match err { + VintedWrapperError::ItemNumberError => unreachable!(), + VintedWrapperError::ItemError(_, _, _) => unreachable!(), + VintedWrapperError::CookiesError(_) => (), + VintedWrapperError::SerdeError(_) => unreachable!(), + VintedWrapperError::ReqWestError(_) => unreachable!(), + } + } }; } @@ -184,19 +204,23 @@ async fn test_get_items_by_size_no_db() { assert!(ok); } - Err(err) => match err { - VintedWrapperError::ItemNumberError => unreachable!(), - VintedWrapperError::ItemError(_, _, _) => (), - VintedWrapperError::CookiesError(_) => (), - VintedWrapperError::SerdeError(_) => (), - }, + Err(err) => { + log::error!("{:#?}", err); + match err { + VintedWrapperError::ItemNumberError => unreachable!(), + VintedWrapperError::ItemError(_, _, _) => unreachable!(), + VintedWrapperError::CookiesError(_) => (), + VintedWrapperError::SerdeError(_) => unreachable!(), + VintedWrapperError::ReqWestError(_) => unreachable!(), + } + } }; } #[tokio::test] async fn test_get_items_by_size() { let vinted = VintedWrapper::new(); - let db: DbController = DbController::new(DB_URL, POOL_SIZE, NoTls).await.unwrap(); + let db: DbController = DbController::new(&DB_URI, POOL_SIZE, NoTls).await.unwrap(); let size = db .get_size_by_title_and_type( &String::from("ES"), @@ -220,12 +244,16 @@ async fn test_get_items_by_size() { assert!(ok); } - Err(err) => match err { - VintedWrapperError::ItemNumberError => unreachable!(), - VintedWrapperError::ItemError(_, _, _) => (), - VintedWrapperError::CookiesError(_) => (), - VintedWrapperError::SerdeError(_) => (), - }, + Err(err) => { + log::error!("{:#?}", err); + match err { + VintedWrapperError::ItemNumberError => unreachable!(), + VintedWrapperError::ItemError(_, _, _) => unreachable!(), + VintedWrapperError::CookiesError(_) => (), + VintedWrapperError::SerdeError(_) => unreachable!(), + VintedWrapperError::ReqWestError(_) => unreachable!(), + } + } }; } @@ -244,12 +272,16 @@ async fn test_get_items_by_material() { Ok(items) => { assert!(items.items.len() <= num); } - Err(err) => match err { - VintedWrapperError::ItemNumberError => unreachable!(), - VintedWrapperError::ItemError(_, _, _) => (), - VintedWrapperError::CookiesError(_) => (), - VintedWrapperError::SerdeError(_) => (), - }, + Err(err) => { + log::error!("{:#?}", err); + match err { + VintedWrapperError::ItemNumberError => unreachable!(), + VintedWrapperError::ItemError(_, _, _) => unreachable!(), + VintedWrapperError::CookiesError(_) => (), + VintedWrapperError::SerdeError(_) => unreachable!(), + VintedWrapperError::ReqWestError(_) => unreachable!(), + } + } }; } @@ -272,12 +304,16 @@ async fn test_get_items_by_color() { Ok(items) => { assert!(items.items.len() <= num); } - Err(err) => match err { - VintedWrapperError::ItemNumberError => unreachable!(), - VintedWrapperError::ItemError(_, _, _) => (), - VintedWrapperError::CookiesError(_) => (), - VintedWrapperError::SerdeError(_) => (), - }, + Err(err) => { + log::error!("{:#?}", err); + match err { + VintedWrapperError::ItemNumberError => unreachable!(), + VintedWrapperError::ItemError(_, _, _) => unreachable!(), + VintedWrapperError::CookiesError(_) => (), + VintedWrapperError::SerdeError(_) => unreachable!(), + VintedWrapperError::ReqWestError(_) => unreachable!(), + } + } }; } @@ -302,21 +338,23 @@ async fn test_get_items_by_currency() { assert!(ok); } - Err(err) => match err { - VintedWrapperError::ItemNumberError => unreachable!(), - VintedWrapperError::ItemError(_, _, _) => (), - VintedWrapperError::CookiesError(_) => (), - VintedWrapperError::SerdeError(_) => (), - }, + Err(err) => { + log::error!("{:#?}", err); + match err { + VintedWrapperError::ItemNumberError => unreachable!(), + VintedWrapperError::ItemError(_, _, _) => unreachable!(), + VintedWrapperError::CookiesError(_) => (), + VintedWrapperError::SerdeError(_) => unreachable!(), + VintedWrapperError::ReqWestError(_) => unreachable!(), + } + } }; } #[tokio::test] async fn test_get_advanced_items() { env_logger::builder().is_test(true).init(); - let db = DbController::new("postgres://postgres:postgres@localhost/vinted-rs", 5, NoTls) - .await - .unwrap(); + let db = DbController::new(&DB_URI, 5, NoTls).await.unwrap(); let adidas = db.get_brand_by_name(&"Adidas").await.unwrap(); let nike = db.get_brand_by_name(&"Nike").await.unwrap(); @@ -335,19 +373,34 @@ async fn test_get_advanced_items() { Ok(items) => { if !items.items.is_empty() { for item in items.items { - let advanced = vinted - .get_advanced_item(item.id, None, None, None) - .await - .unwrap(); - assert_eq!(item.id, advanced.id); + let raw = vinted.get_advanced_item(item.id, None, None, None).await; + match raw { + Ok(advanced) => { + assert_eq!(item.id, advanced.id); + } + Err(err) => { + log::error!("{:#?}", err); + match err { + VintedWrapperError::ItemNumberError => unreachable!(), + VintedWrapperError::ItemError(_, _, _) => (), + VintedWrapperError::CookiesError(_) => (), + VintedWrapperError::SerdeError(_) => unreachable!(), + VintedWrapperError::ReqWestError(_) => unreachable!(), + } + } + } } } } - Err(err) => match err { - VintedWrapperError::ItemNumberError => unreachable!(), - VintedWrapperError::SerdeError(_) => (), - VintedWrapperError::ItemError(_, _, _) => (), - VintedWrapperError::CookiesError(_) => (), - }, + Err(err) => { + log::error!("{:#?}", err); + match err { + VintedWrapperError::ItemNumberError => unreachable!(), + VintedWrapperError::ItemError(_, _, _) => unreachable!(), + VintedWrapperError::CookiesError(_) => (), + VintedWrapperError::SerdeError(_) => unreachable!(), + VintedWrapperError::ReqWestError(_) => unreachable!(), + } + } }; }