Skip to content

Commit

Permalink
Merge pull request #373 from stakwork/feat/dynamic-aws
Browse files Browse the repository at this point in the history
Feat/dynamic aws
  • Loading branch information
Evanfeenstra authored Nov 5, 2024
2 parents 3414293 + 316f5d1 commit 23eb84c
Show file tree
Hide file tree
Showing 15 changed files with 338 additions and 186 deletions.
4 changes: 0 additions & 4 deletions config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,6 @@ services:
# So that Traefik can listen to the Docker events
- /var/run/docker.sock:/var/run/docker.sock
- /home/admin/letsencrypt:/letsencrypt
environment:
- AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
- AWS_REGION=$AWS_REGION
logging:
options:
max-size: 10m
Expand Down
8 changes: 1 addition & 7 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,6 @@ services:
# So that Traefik can listen to the Docker events
- /var/run/docker.sock:/var/run/docker.sock
- /home/admin/letsencrypt:/letsencrypt
environment:
- AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
- AWS_REGION=$AWS_REGION
logging:
options:
max-size: 10m
Expand Down Expand Up @@ -73,10 +69,7 @@ services:
- NETWORK=bitcoin
- ROCKET_ADDRESS=0.0.0.0
- ROCKET_PORT=8000
- AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
- AWS_REGION=$AWS_REGION
- AWS_S3_REGION_NAME=$AWS_S3_REGION_NAME
- STAKWORK_ADD_NODE_TOKEN=$STAKWORK_ADD_NODE_TOKEN
- STAKWORK_RADAR_REQUEST_TOKEN=$STAKWORK_RADAR_REQUEST_TOKEN
- CLN_MAINNET_BTC=$CLN_MAINNET_BTC
Expand All @@ -92,6 +85,7 @@ services:
- SUPER_URL=$SUPER_URL
- SUPER_TOKEN=$SUPER_TOKEN
- QUESTION_AND_ANSWER_WORKFLOW_ID=$QUESTION_AND_ANSWER_WORKFLOW_ID
- AWS_S3_BUCKET_NAME=$AWS_S3_BUCKET_NAME

networks:
sphinx-swarm:
Expand Down
8 changes: 1 addition & 7 deletions second-brain.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,6 @@ services:
# So that Traefik can listen to the Docker events
- /var/run/docker.sock:/var/run/docker.sock
- /home/admin/letsencrypt:/letsencrypt
environment:
- AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
- AWS_REGION=$AWS_REGION
logging:
options:
max-size: 10m
Expand Down Expand Up @@ -70,10 +66,7 @@ services:
- NETWORK=bitcoin
- ROCKET_ADDRESS=0.0.0.0
- ROCKET_PORT=8800
- AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
- AWS_REGION=$AWS_REGION
- AWS_S3_REGION_NAME=$AWS_S3_REGION_NAME
- STAKWORK_ADD_NODE_TOKEN=$STAKWORK_ADD_NODE_TOKEN
- STAKWORK_RADAR_REQUEST_TOKEN=$STAKWORK_RADAR_REQUEST_TOKEN
- SECOND_BRAIN_ONLY=true
Expand Down Expand Up @@ -105,6 +98,7 @@ services:
- SUPER_URL=$SUPER_URL
- LOCAL_LLAMA=$LOCAL_LLAMA
- QUESTION_AND_ANSWER_WORKFLOW_ID=$QUESTION_AND_ANSWER_WORKFLOW_ID
- AWS_S3_BUCKET_NAME=$AWS_S3_BUCKET_NAME

networks:
sphinx-swarm:
Expand Down
5 changes: 1 addition & 4 deletions sphinx.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,6 @@ services:
# So that Traefik can listen to the Docker events
- /var/run/docker.sock:/var/run/docker.sock
- /home/admin/letsencrypt:/letsencrypt
environment:
- AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
- AWS_REGION=$AWS_REGION
logging:
options:
max-size: 10m
Expand Down Expand Up @@ -76,6 +72,7 @@ services:
- TRIBES_HOST=$TRIBES_HOST
- SUPER_TOKEN=$SUPER_TOKEN
- SUPER_URL=$SUPER_URL
- AWS_S3_BUCKET_NAME=$AWS_S3_BUCKET_NAME

networks:
sphinx-swarm:
Expand Down
7 changes: 2 additions & 5 deletions sphinxv2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,6 @@ services:
# So that Traefik can listen to the Docker events
- /var/run/docker.sock:/var/run/docker.sock
- /home/admin/letsencrypt:/letsencrypt
environment:
- AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
- AWS_REGION=$AWS_REGION
logging:
options:
max-size: 10m
Expand Down Expand Up @@ -75,7 +71,8 @@ services:
- IS_ROUTER=$IS_ROUTER
- SEED=$SEED
- CLN_MAINNET_BTC=$CLN_MAINNET_BTC
- AWS_S3_REGION_NAME=$AWS_S3_REGION_NAME
- AWS_REGIONE=$AWS_REGION
- AWS_S3_BUCKET_NAME=$AWS_S3_BUCKET_NAME

networks:
sphinx-swarm:
Expand Down
38 changes: 30 additions & 8 deletions src/backup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use crate::utils::{domain, getenv};
use anyhow::{Context, Result};
use aws_config::meta::region::RegionProviderChain;
use aws_config::Region;
use aws_sdk_ec2::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::operation::create_multipart_upload::CreateMultipartUploadOutput;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Delete, ObjectIdentifier};
use aws_sdk_s3::Client;
Expand Down Expand Up @@ -51,6 +52,8 @@ pub async fn backup_containers(backup_services: Vec<String>) -> Result<()> {

let mut containers: Vec<(String, String, String)> = Vec::new();

log::info!("About to start get backup containers");

for node in nodes.iter() {
let node_name = node.name();
let hostname = domain(&node_name);
Expand All @@ -64,6 +67,8 @@ pub async fn backup_containers(backup_services: Vec<String>) -> Result<()> {
}
}

log::info!("Containers to be backed up: {:?}", containers);

download_and_zip_from_container(containers).await?;

Ok(())
Expand Down Expand Up @@ -179,10 +184,10 @@ async fn upload_to_s3_multi(bucket: &str, key: &str) -> Result<bool> {
const MAX_CHUNKS: u64 = 10000;

// Read the custom region environment variable
let region = match getenv("AWS_S3_REGION_NAME") {
let region = match getenv("AWS_REGION") {
Ok(value) => value,
Err(_msg) => {
log::error!("AWS_S3_REGION_NAME is not provided in environment variable");
log::error!("AWS_REGION is not provided in environment variable");
return Ok(false);
}
};
Expand All @@ -198,12 +203,22 @@ async fn upload_to_s3_multi(bucket: &str, key: &str) -> Result<bool> {
.await;
let client = Client::new(&config);

let multipart_upload_res: CreateMultipartUploadOutput = client
let result = client
.create_multipart_upload()
.bucket(bucket)
.key(key)
.send()
.await?;
.await;

// CreateMultipartUploadOutput

let multipart_upload_res = match result {
Ok(response) => response,
Err(err) => {
log::error!("Error creating multipart: {:?}", err);
return Ok(false);
}
};

let upload_id = match multipart_upload_res.upload_id() {
Some(id) => id,
Expand Down Expand Up @@ -267,7 +282,7 @@ async fn upload_to_s3_multi(bucket: &str, key: &str) -> Result<bool> {
{
Ok(res) => res,
Err(e) => {
log::error!("Error uploading part: {}", e);
log::error!("Error uploading part: {:?}", e);
return Ok(false);
}
};
Expand All @@ -283,22 +298,29 @@ async fn upload_to_s3_multi(bucket: &str, key: &str) -> Result<bool> {
.set_parts(Some(upload_parts))
.build();

let _complete_multipart_upload_res = client
let _complete_multipart_upload_res = match client
.complete_multipart_upload()
.bucket(bucket)
.key(key)
.multipart_upload(completed_multipart_upload)
.upload_id(upload_id)
.send()
.await?;
.await
{
Ok(res) => res,
Err(err) => {
log::error!("Error completing multipart: {:?}", err);
return Ok(false);
}
};

Ok(true)
}

// Deletes old backups from the S3 bucket
pub async fn delete_old_backups(bucket: &str, retention_days: i64) -> Result<()> {
// Read the custom region environment variable
let region = getenv("AWS_S3_REGION_NAME")?;
let region = getenv("AWS_REGION")?;

// Create a region provider chain
let region_provider = RegionProviderChain::first_try(Some(Region::new(region)));
Expand Down
3 changes: 1 addition & 2 deletions src/bin/stack/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,7 @@ echo 'HOST=swarm<ENTER_NUMBER>.sphinx.chat' >> .env &&
echo 'NETWORK=bitcoin' >> .env &&
echo 'AWS_ACCESS_KEY_ID=<ENTER_AWS_ACCESS_KEY>' >> .env &&
echo 'AWS_SECRET_ACCESS_KEY=<ENTER_AWS_SECRET_KEY>' >> .env &&
echo 'AWS_REGION=us-east-1a' >> .env &&
echo 'AWS_S3_REGION_NAME=us-east-1' >> .env &&
echo 'AWS_REGION=us-east-1' >> .env &&
echo 'STAKWORK_ADD_NODE_TOKEN=<ENTER_STAKWORK_TOKEN>' >> .env &&
echo 'STAKWORK_RADAR_REQUEST_TOKEN=<ENTER_STAKWORK_TOKEN>' >> .env &&
echo 'NO_REMOTE_SIGNER=true' >> .env &&
Expand Down
9 changes: 8 additions & 1 deletion src/bin/super/aws_util.rs
Original file line number Diff line number Diff line change
@@ -1,16 +1,23 @@
use anyhow::Error;
use aws_config::meta::region::RegionProviderChain;
use aws_config::timeout::TimeoutConfig;
use aws_config::Region;
use aws_sdk_ec2::Client;
use aws_smithy_types::retry::RetryConfig;
use sphinx_swarm::utils::getenv;
use tokio::time::Duration;

pub async fn make_aws_client() -> Result<Client, Error> {
let region = getenv("AWS_S3_REGION_NAME")?;
let region = getenv("AWS_REGION")?;
let region_provider = RegionProviderChain::first_try(Some(Region::new(region)));
let timeout_config = TimeoutConfig::builder()
.connect_timeout(Duration::from_secs(5))
.read_timeout(Duration::from_secs(60))
.build();
let config = aws_config::from_env()
.region(region_provider)
.retry_config(RetryConfig::standard().with_max_attempts(10))
.timeout_config(timeout_config)
.load()
.await;

Expand Down
45 changes: 36 additions & 9 deletions src/bin/super/route53.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use anyhow::{anyhow, Error};
use aws_config::meta::region::RegionProviderChain;
use aws_config::Region;
use aws_sdk_ec2::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_route53::types::{
Change, ChangeAction, ChangeBatch, ResourceRecord, ResourceRecordSet,
};
Expand All @@ -12,7 +13,7 @@ pub async fn add_domain_name_to_route53(
domain_names: Vec<&str>,
public_ip: &str,
) -> Result<(), Error> {
let region = getenv("AWS_S3_REGION_NAME")?;
let region = getenv("AWS_REGION")?;
let hosted_zone_id = getenv("ROUTE53_ZONE_ID")?;
let region_provider = RegionProviderChain::first_try(Some(Region::new(region)));
let config = aws_config::from_env()
Expand Down Expand Up @@ -50,18 +51,44 @@ pub async fn add_domain_name_to_route53(
.build()
.map_err(|err| anyhow!(err.to_string()))?;

let response = route53_client
let result = route53_client
.change_resource_record_sets()
.hosted_zone_id(hosted_zone_id)
.change_batch(change_batch)
.send()
.await?;
.await;

log::info!(
"Route 53 change status for {:?}: {:?}",
domain_names,
response.change_info()
);
match result {
Ok(response) => {
log::info!(
"Route 53 change status for {:?}: {:?}",
domain_names,
response.change_info()
);

Ok(())
return Ok(());
}
Err(SdkError::ServiceError(service_error)) => {
let err = service_error
.err()
.message()
.unwrap_or("Unknown error")
.to_string();
log::error!("Service error: {}", err);
return Err(anyhow!(err));
}
Err(SdkError::TimeoutError(_)) => {
let err_msg = "Request timed out.";
log::error!("{}", err_msg);
return Err(anyhow!(err_msg));
}
Err(SdkError::DispatchFailure(err)) => {
log::error!("Network error: {:?}", err);
return Err(anyhow!("Network error"));
}
Err(e) => {
log::error!("Unexpected error: {:?}", e);
return Err(anyhow!("Unexpected error"));
}
}
}
Loading

0 comments on commit 23eb84c

Please sign in to comment.