From 7ce2553ee48640c6416f1e0455a36ce181fa610e Mon Sep 17 00:00:00 2001
From: Falko Zurell <falko.zurell@here.com>
Date: Mon, 3 Mar 2025 22:41:36 +0100
Subject: [PATCH] first compiled version if rewrite
---
.gitignore | 1 +
Cargo.toml | 10 +-
src/image_description.rs | 101 ++++++++++++++++
src/main.rs | 240 ++++++++++++++++-----------------------
src/pixelfed.rs | 102 +++++++++++++++++
5 files changed, 307 insertions(+), 147 deletions(-)
create mode 100644 src/image_description.rs
create mode 100644 src/pixelfed.rs
diff --git a/.gitignore b/.gitignore
index 4428080..3d09459 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,4 @@
Cargo.lock
config.json
.netrc
+config.json.instagram
diff --git a/Cargo.toml b/Cargo.toml
index a2de8aa..7cc5cd8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -7,8 +7,10 @@ edition = "2021"
rustflags = ["--out-dir", "target/output"]
[dependencies]
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-reqwest = { version = "0.11", features = ["blocking", "json", "multipart"] }
+serde = { version = "1.0.218", features = ["derive"] }
+serde_json = "1.0.140"
+reqwest = { version = "0.12", features = ["blocking", "json", "multipart", "stream"] }
tokio = { version = "1.0", features = ["full"] }
-base64 = "0.21"
+base64 = "0.22.1"
+clap = { version = "4.5.3", features = ["derive"] }
+
diff --git a/src/image_description.rs b/src/image_description.rs
new file mode 100644
index 0000000..adee03b
--- /dev/null
+++ b/src/image_description.rs
@@ -0,0 +1,101 @@
+use base64::{Engine as _, engine::general_purpose::STANDARD};
+
+
+// module to hold all code for generating/fetching image descriptions
+// Input is the image name
+// Output is a String containing the image description
+pub struct ChatGPTConfig {
+ pub openai_api_key: String,
+ pub openai_api_url: String,
+ pub openai_model: String
+
+}
+
+pub struct OllamaConfig {
+ pub ollama_api_key: String,
+ pub ollama_api_url: String,
+ pub ollama_model: String
+}
+
+pub struct FileConfig {
+ pub caption_extension: String,
+}
+
+// fetch the imagedescription from a file named like the Image
+pub async fn get_description_from_file(image_name: String , file_config: FileConfig) -> Result<String, Box<dyn super::Error>> {
+ //read image caption from a local file that
+ //has the same name than the image with the extension ".caption.txt"
+ let caption_extension = file_config.caption_extension;
+ let captionname = format!("{}{}", image_name, caption_extension);
+
+ println!("Looking for {}",captionname);
+ let caption_data = tokio::fs::read_to_string(captionname).await.map_err(|e| format!("Failed to read caption from file: {}", e))?;
+
+ Ok(caption_data)
+}
+
+// fetch image description from ChatGPT
+pub async fn get_description_from_chatgpt(image_name: String, chatgpt_config: self::ChatGPTConfig) -> Result<String, Box<dyn super::Error>> {
+ // Read and encode image
+ let image_data = tokio::fs::read(image_name)
+ .await
+ .map_err(|e| format!("Failed to read image file: {}", e))?;
+
+
+ // Base64 encode the image for ChatGTP API
+ let base64_image = STANDARD.encode(image_data);
+
+ // Create ChatGPT API request
+ let client = reqwest::Client::new();
+ let response = client
+ .post(chatgpt_config.openai_api_url)
+ .header("Authorization", format!("Bearer {}", chatgpt_config.openai_api_key))
+ .header("Content-Type", "application/json")
+ .json(&super::json!({
+ "model": chatgpt_config.openai_model,
+ "max_tokens": 300,
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Please describe this image concisely for use as an alt text description. Focus on key visual elements and context."
+ },
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": format!("data:image/jpeg;base64,{}", base64_image)
+ }
+ }
+ ]
+ }
+ ]
+ }))
+ .send()
+ .await?;
+
+ // Improved error handling for API response
+ if !response.status().is_success() {
+ let error_text = response.text().await?;
+ return Err(format!("OpenAI API error: {}", error_text).into());
+ }
+
+ let result: super::Value = response.json().await?;
+
+ // More detailed error handling for JSON parsing
+ let description = result["choices"]
+ .get(0)
+ .ok_or("No choices in response")?
+ ["message"]["content"]
+ .as_str()
+ .ok_or("Invalid content format in response")?
+ .to_string();
+
+ Ok(description)
+}
+
+// fetch images description from own OLLAMA server
+pub async fn get_description_from_ollama(image_name: String, ollama_config: OllamaConfig) -> Result<String, Box<dyn super::Error>> {
+ Ok("Not implemented yet".to_string())
+}
diff --git a/src/main.rs b/src/main.rs
index eafd7d7..5ac94da 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -4,27 +4,80 @@ use serde_json::{json, Value};
use reqwest::blocking::Client;
use reqwest::{self};
use std::error::Error;
+use std::fs::File;
+use std::io::BufReader;
+use clap::{Parser, ValueEnum};
use base64::{Engine as _, engine::general_purpose::STANDARD};
+use std::path::PathBuf;
+
+mod pixelfed;
+pub mod image_description;
+
+#[derive(Parser)]
+#[command(name = "Pixelfed Image Bulk Uploader")]
+#[command(version = "1.0")]
+#[command(about = "Bulk uploads images to Pixelfed with image descriptions", long_about = None)]
+#[command(version, about, long_about = None)]
+struct Cli {
+ /// Image description mode
+ #[arg(short, long, default_value = "file")]
+ mode: Mode,
+ /// The title of the posting
+ #[arg(short, long)]
+ title: String,
+ /// The path to the file to read
+ #[arg(short, long)]
+ image_path: String,
+ /// Sets a custom config file
+ #[arg(short, long, value_name = "FILE")]
+ config: Option<String>,
+}
+
+
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)]
+enum Mode {
+ /// Use ChatGTP
+ ChatGPT,
+ /// Taking from a file
+ File,
+ /// Local LLM
+ Local,
+}
+
#[derive(Debug, Deserialize)]
struct Config {
pixelfed_url: String,
- access_token: String,
- visibility: String, // Should be "unlisted"
- default_text: String,
- batch_size: usize,
+ pixelfed_access_token: String,
+ pixelfed_visibility: String, // Should be "unlisted"
+ pixelfed_default_text: String,
+ pixelfed_batch_size: usize,
openai_api_key: String,
openai_api_url: String,
- openai_model: String
+ openai_model: String,
+ ollama_api_key: String,
+ ollama_api_url: String,
+ ollama_model: String,
+ caption_extension: String,
}
-fn load_config() -> Result<Config, Box<dyn Error>> {
- let config_str = fs::read_to_string("config.json")?;
- let config: Config = serde_json::from_str(&config_str)?;
+fn load_config(config_file: String) -> Result<Config, Box<dyn Error>> {
+ //let config_str = fs::read_to_string("config.json")?;
+ // Open the file in read-only mode with buffer.
+
+ let file = File::open(PathBuf::from(config_file))?;
+ let reader = BufReader::new(file);
+
+ // Read the JSON contents of the file as an instance of `User`.
+
+
+ let config: Config = serde_json::from_reader(reader)?;
Ok(config)
}
+
+// get all the JPEG files from the give directory
fn get_jpeg_files(directory: &str) -> Vec<String> {
let mut images = Vec::new();
if let Ok(entries) = fs::read_dir(directory) {
@@ -40,149 +93,50 @@ fn get_jpeg_files(directory: &str) -> Vec<String> {
images
}
-async fn get_image_description(config: &Config, image_path: &String) -> Result<String, Box<dyn Error>> {
- // Read and encode image
- let image_data = tokio::fs::read(image_path)
- .await
- .map_err(|e| format!("Failed to read image file: {}", e))?;
-
- let base64_image = STANDARD.encode(&image_data);
-
- // Create ChatGPT API request
- let client = reqwest::Client::new();
- let response = client
- .post(&config.openai_api_url)
- .header("Authorization", format!("Bearer {}", config.openai_api_key))
- .header("Content-Type", "application/json")
- .json(&json!({
- "model": config.openai_model,
- "max_tokens": 300,
- "messages": [
- {
- "role": "user",
- "content": [
- {
- "type": "text",
- "text": "Please describe this image concisely for use as an alt text description. Focus on key visual elements and context."
- },
- {
- "type": "image_url",
- "image_url": {
- "url": format!("data:image/jpeg;base64,{}", base64_image)
- }
- }
- ]
- }
- ]
- }))
- .send()
- .await?;
-
- // Improved error handling for API response
- if !response.status().is_success() {
- let error_text = response.text().await?;
- return Err(format!("OpenAI API error: {}", error_text).into());
- }
-
- let result: Value = response.json().await?;
-
- // More detailed error handling for JSON parsing
- let description = result["choices"]
- .get(0)
- .ok_or("No choices in response")?
- ["message"]["content"]
- .as_str()
- .ok_or("Invalid content format in response")?
- .to_string();
-
- Ok(description)
-}
-
-
-fn format_post_text(template: &str, batch_num: usize, total_batches: usize, title: &str) -> String {
- template
- .replace("@batch@", &format!("Batch {} out of {}", batch_num, total_batches))
- .replace("@title@", title)
-}
-
-async fn upload_images_batch(client: &Client, config: &Config, images: &[String], batch_num: usize, total_batches: usize, title: &str) -> Result<(), Box<dyn Error>> {
- let url = format!("{}/api/v1/media", config.pixelfed_url);
- let mut media_ids = Vec::new();
- let mut media_descriptions = Vec::new();
-
- for image_path in images {
- println!("Fetching image description from OpenAI for {}", image_path.to_string());
- let image_description: String ;
- if !config.openai_api_key.is_empty() {
- image_description = get_image_description(&config, &image_path).await?;
- println!("DESC:\n {} \n", &image_description);
- media_descriptions.push(image_description.clone());
-
- } else {
- image_description = String::new();
- }
-
-
- println!("Uploading image {}", image_path.to_string());
-
- let form = reqwest::blocking::multipart::Form::new().text("description", image_description.clone())
- .file("file", image_path)?;
-
- let res = client.post(&url)
- .bearer_auth(&config.access_token)
- .multipart(form)
- .send()?;
-
- let json: serde_json::Value = res.json()?;
- if let Some(id) = json["id"].as_str() {
- media_ids.push(id.to_string());
- }
- }
-
- if !media_ids.is_empty() {
- let post_url = format!("{}/api/v1/statuses", config.pixelfed_url);
- let post_text = format_post_text(&config.default_text, batch_num, total_batches, title);
- let body = serde_json::json!({
- "status": post_text,
- "media_ids": media_ids,
- "alt_texts": media_descriptions,
- "visibility": config.visibility,
- });
-
- println!("Posting batch {} out of {} with media {}", batch_num, total_batches, media_ids.len());
- client.post(&post_url)
- .bearer_auth(&config.access_token)
- .json(&body)
- .send()?;
- }
-
- Ok(())
-}
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
- let args: Vec<String> = std::env::args().collect();
- if args.len() < 2 {
- eprintln!("Usage: {} <directory> [--title <title>]", args[0]);
- std::process::exit(1);
- }
+ let args = Cli::parse();
+
+ //if args.len() < 2 || args.len() > 3 {
+ // eprintln!("Usage: {} <directory> [--title <title>]", args[0]);
+ // eprintln!("Usage: {} <directory> -ready [--title <title>]", args[0]);
+ // std::process::exit(1);
+ //}
- let mut title = "".to_string();
- if let Some(index) = args.iter().position(|x| x == "--title") {
- if index + 1 < args.len() {
- title = args[index + 1].clone();
- }
+ let title = args.title;
+ let mut my_config: String;
+
+
+
+ match args.config {
+ Some(configstring) => { my_config = configstring},
+ None => {my_config = "config.json".to_string()},
}
+ println!("effictive config file: {}", my_config);
- let config = load_config()?;
- let images = get_jpeg_files(&args[1]);
- let client = Client::new();
- let total_batches = (images.len() + config.batch_size - 1) / config.batch_size;
+
+ let config = load_config(my_config).unwrap();
+ println!("Config OK? true");
+
+
+
+
+
+
+ // get list of all the images in the gives path
+ let images = get_jpeg_files(&args.image_path);
+ println!("Images empty? {}", images.is_empty().to_string());
+
+ // knowing now the total number of images, calculate the number of batches
+ let total_batches = (images.len() + config.pixelfed_batch_size - 1) / config.pixelfed_batch_size;
println!("Found a total of {} images to upload. Will take {} batches", images.len(), total_batches);
- for (i, chunk) in images.chunks(config.batch_size).enumerate() {
- upload_images_batch(&client, &config, chunk, i + 1, total_batches, &title).await?;
+
+ // now iterate over all images in batches of batch_size
+ for (i, chunk) in images.chunks(config.pixelfed_batch_size).enumerate() {
+ println!("{}", i.clone());
+ pixelfed::bulk_upload_images(&config, chunk, i + 1, total_batches, &title, &args.mode).await?;
}
-
println!("All images uploaded successfully.");
Ok(())
}
diff --git a/src/pixelfed.rs b/src/pixelfed.rs
new file mode 100644
index 0000000..2efb56e
--- /dev/null
+++ b/src/pixelfed.rs
@@ -0,0 +1,102 @@
+use reqwest::{self, multipart};
+use std::error::Error;
+
+struct PixelfedConfig {
+ pixelfed_url: String,
+ pixelfed_access_token: String,
+ pixelfed_visibility: String, // Should be "unlisted"
+ pixelfed_default_text: String,
+ pixelfed_batch_size: usize,
+}
+
+fn format_post_text(template: &str, batch_num: usize, total_batches: usize, title: &str) -> String {
+ template
+ .replace("@batch@", &format!("Batch {} out of {}", batch_num, total_batches))
+ .replace("@title@", title)
+}
+
+// upload a single image to pixelfed
+pub async fn bulk_upload_images(config: &super::Config, images: &[String], batch_num: usize, total_batches: usize, title: &str, caption_mode: &super::Mode) -> Result<(), Box<dyn Error>> {
+ let mut media_ids = Vec::new();
+ let mut media_descriptions = Vec::new();
+ let client = reqwest::Client::new();
+ let pxl_config = PixelfedConfig {
+ pixelfed_url: config.pixelfed_url.clone(),
+ pixelfed_access_token: config.pixelfed_access_token.clone(),
+ pixelfed_visibility: config.pixelfed_visibility.clone(),
+ pixelfed_default_text: config.pixelfed_default_text.clone(),
+ pixelfed_batch_size: config.pixelfed_batch_size,
+ };
+ let url = format!("{}/api/v1/media", config.pixelfed_url.clone());
+
+ for image_path in images {
+ let description: String;
+ println!("Uploading image {}", image_path.to_string());
+ // get image description depending on the caption_mode
+ match caption_mode {
+ super::Mode::ChatGPT => {
+ let im_config = super::image_description::ChatGPTConfig {
+ openai_model: config.openai_model.clone(),
+ openai_api_key: config.openai_api_key.clone(),
+ openai_api_url: config.openai_api_url.clone(),
+ };
+ description = super::image_description::get_description_from_chatgpt(image_path.to_string(), im_config).await?;
+ media_descriptions.push(description.clone());
+ },
+ super::Mode::File => {
+ let im_config = super::image_description::FileConfig {
+ caption_extension: config.caption_extension.clone(),
+ };
+ println!("Fetching image description from File for {}", image_path.to_string());
+ description = super::image_description::get_description_from_file(image_path.to_string(), im_config).await?;
+ media_descriptions.push(description.clone());
+ },
+ super::Mode::Local => {
+ let im_config = super::image_description::OllamaConfig {
+ ollama_api_key: config.ollama_api_key.clone(),
+ ollama_api_url: config.ollama_api_url.clone(),
+ ollama_model: config.ollama_model.clone(),
+ };
+ println!("Fetching image description from OLLAMA for {}", image_path.to_string());
+ description = super::image_description::get_description_from_ollama(image_path.to_string(), im_config).await?;
+ media_descriptions.push(description.clone());
+ },
+ }
+
+ println!("Uploading image {}", image_path.to_string());
+
+ // construct the upload form for Pixelfed Upload of a single image including image description
+ let form = multipart::Form::new()
+ .file("file", image_path).await?;
+ // upload the form to Pixelfed
+ let res = client.post(&url)
+ .bearer_auth(&config.pixelfed_access_token)
+ .multipart(form)
+ .send()
+ .await?;
+
+ let res_json: serde_json::Value = res.json().await?;
+ let image_id = res_json["id"].as_str().unwrap().to_string();
+ media_ids.push(image_id);
+ }
+
+ if !media_ids.is_empty() {
+ let post_url = format!("{}/api/v1/statuses", config.pixelfed_url);
+ let post_text = format_post_text(&config.pixelfed_default_text, batch_num, total_batches, title);
+ let body = serde_json::json!({
+ "status": post_text,
+ "media_ids": media_ids,
+ "alt_texts": media_descriptions,
+ "visibility": config.pixelfed_visibility,
+ });
+ println!("Body: \n{}", body.to_string());
+ println!("MediaIDs: {}", media_ids.len());
+ println!("Alt_texts: {}", media_descriptions.len());
+ println!("Posting batch {} out of {} with media {}", batch_num, total_batches, media_ids.len());
+ client.post(&post_url)
+ .bearer_auth(&config.pixelfed_access_token)
+ .json(&body)
+ .send().await?;
+ }
+ Ok(())
+}