Raven/osirose-new
SHA256
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 6 Releases Wiki Activity

consul_services_update #9

Merged
Raven merged 3 commits from consul_services_update into characters 2025-03-12 22:20:08 -04:00
Conversation 0 Commits 3 Files Changed 12 +307 -34
12 changed files with 307 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
api-service/src/axum_gateway.rs
View File

@@ -8,6 +8,7 @@ use std::sync::Arc;
use tonic::transport::Channel;
use tower_http::cors::{Any, CorsLayer};
use crate::axum_gateway::auth::LogoutRequest;
use auth::auth_service_client::AuthServiceClient;
use auth::{LoginRequest, RegisterRequest};
use log::{error, info};
@@ -32,6 +33,16 @@ struct RestLoginResponse {
session_id: String,
}
#[derive(Serialize, Deserialize)]
struct RestLogoutRequest {
session_id: String,
}
#[derive(Serialize, Deserialize)]
struct RestLogoutResponse {
message: String,
}
#[derive(Serialize, Deserialize)]
struct RestRegisterRequest {
username: String,
@@ -52,8 +63,6 @@ async fn login_handler(
) -> Result<Json<RestLoginResponse>, axum::http::StatusCode> {
let ip_address = addr.ip().to_string();
info!("Client IP Address: {}", ip_address);
let request = tonic::Request::new(LoginRequest {
username: payload.username.clone(),
password: payload.password.clone(),
@@ -75,6 +84,28 @@ async fn login_handler(
}
}
async fn logout_handler(
ConnectInfo(addr): ConnectInfo<SocketAddr>,
State(grpc_client): State<Arc<Mutex<AuthServiceClient<Channel>>>>,
Json(payload): Json<RestLogoutRequest>,
) -> Result<axum::http::StatusCode, axum::http::StatusCode> {
let ip_address = addr.ip().to_string();
let request = tonic::Request::new(LogoutRequest {
session_id: payload.session_id
});
let mut client = grpc_client.lock().await; // Lock the mutex to get mutable access
match client.logout(request).await {
Ok(_response) => {
Ok(axum::http::StatusCode::OK)
}
Err(_e) => {
Err(axum::http::StatusCode::INTERNAL_SERVER_ERROR)
}
}
}
async fn register_handler(
ConnectInfo(addr): ConnectInfo<SocketAddr>,
State(grpc_client): State<Arc<Mutex<AuthServiceClient<Channel>>>>,
@@ -110,6 +141,7 @@ pub async fn serve_rest_api(
let app = Router::new()
.route("/api/login", post(login_handler))
.route("/api/logout", post(logout_handler))
.route("/api/register", post(register_handler))
.with_state(grpc_client)
.layer(cors);
@@ -123,8 +155,8 @@ pub async fn serve_rest_api(
listener,
app.into_make_service_with_connect_info::<SocketAddr>(),
)
.await
.unwrap();
.await
.unwrap();
Ok(())
}

52
auth-service/src/main.rs
View File

@@ -9,9 +9,10 @@ use std::env;
use std::str::FromStr;
use std::sync::Arc;
use tonic::transport::Server;
use tracing::{info, Level};
use tracing::{debug, info, Level};
use utils::consul_registration;
use utils::service_discovery::get_service_address;
use utils::multi_service_load_balancer::{LoadBalancingStrategy, MultiServiceLoadBalancer};
use utils::service_discovery::{get_service_address, get_service_endpoints_by_dns};
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -29,12 +30,39 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let addr = env::var("LISTEN_ADDR").unwrap_or_else(|_| "0.0.0.0".to_string());
let port = env::var("AUTH_SERVICE_PORT").unwrap_or_else(|_| "50051".to_string());
let consul_url = env::var("CONSUL_URL").unwrap_or_else(|_| "http://127.0.0.1:8500".to_string());
let consul_address = env::var("CONSUL_ADDRESS").unwrap_or_else(|_| "127.0.0.1".to_string());
let consul_port = env::var("CONSUL_PORT").unwrap_or_else(|_| "8500".to_string());
let consul_dns_port = env::var("CONSUL_DNS_PORT").unwrap_or_else(|_| "8600".to_string());
let consul_url = format!("http://{}:{}", consul_address, consul_port);
let consul_dns_url = format!("{}:{}", consul_address, consul_dns_port);
let service_name = env::var("SERVICE_NAME").unwrap_or_else(|_| "auth-service".to_string());
let service_address = env::var("AUTH_SERVICE_ADDR").unwrap_or_else(|_| "127.0.0.1".to_string());
let service_port = port.clone();
let db_nodes = get_service_address(&consul_url, "database-service").await?;
let session_nodes = get_service_address(&consul_url, "session-service").await?;
let lb = MultiServiceLoadBalancer::new(&consul_dns_url, LoadBalancingStrategy::RoundRobin);
let mut db_url = "".to_string();
match lb.get_endpoint("database-service", "grpc").await? {
Some(endpoint) => {
db_url = format!("http://{}", endpoint);
},
None => {
println!("No endpoints available for database-service");
}
}
let mut session_service_address = "".to_string();
match lb.get_endpoint("session-service", "grpc").await? {
Some(endpoint) => {
session_service_address = format!("http://{}", endpoint);
},
None => {
println!("No endpoints available for session-service");
}
}
let db_client = Arc::new(DatabaseClient::connect(&db_url).await?);
let session_client = Arc::new(SessionServiceClient::connect(session_service_address).await?);
// Register service with Consul
let service_id = consul_registration::get_or_generate_service_id(env!("CARGO_PKG_NAME"));
@@ -54,20 +82,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
)
.await?;
let db_address = db_nodes.get(0).unwrap();
let db_url = format!(
"http://{}:{}",
db_address.ServiceAddress, db_address.ServicePort
);
let db_client = Arc::new(DatabaseClient::connect(&db_url).await?);
let session_address = session_nodes.get(0).unwrap();
let session_address = format!(
"http://{}:{}",
session_address.ServiceAddress, session_address.ServicePort
);
let session_client = Arc::new(SessionServiceClient::connect(session_address).await?);
let full_addr = format!("{}:{}", &addr, port);
let address = full_addr.parse().expect("Invalid address");
let auth_service = MyAuthService {

49
docker-compose.yml
View File

@@ -5,6 +5,8 @@
dockerfile: Dockerfile
restart: on-failure
container_name: frontend
networks:
- frontend
ports:
- "3000:80"
env_file:
@@ -18,6 +20,8 @@
context: ./
dockerfile: ./auth-service/Dockerfile
restart: on-failure
networks:
- backend
ports:
- "50051:50051"
env_file:
@@ -37,6 +41,9 @@
context: ./
dockerfile: ./api-service/Dockerfile
restart: on-failure
networks:
- frontend
- backend
ports:
- "8080:8080"
- "8081:8081"
@@ -54,6 +61,8 @@
context: ./
dockerfile: ./database-service/Dockerfile
restart: on-failure
networks:
- backend
ports:
- "50052:50052"
env_file:
@@ -70,6 +79,8 @@
context: ./
dockerfile: ./character-service/Dockerfile
restart: on-failure
networks:
- backend
ports:
- "50053:50053"
env_file:
@@ -86,6 +97,8 @@
context: ./
dockerfile: ./world-service/Dockerfile
restart: on-failure
networks:
- backend
ports:
- "50054:50054"
env_file:
@@ -102,9 +115,12 @@
context: ./
dockerfile: ./packet-service/Dockerfile
restart: on-failure
networks:
- backend
ports:
- "29000:29000"
- "4001:4001"
- "8082:8082"
env_file:
- ./packet-service/.env
- .env
@@ -119,6 +135,8 @@
context: ./
dockerfile: ./session-service/Dockerfile
restart: on-failure
networks:
- backend
ports:
- "50055:50055"
env_file:
@@ -134,6 +152,9 @@
env_file:
- .env
restart: on-failure
networks:
backend:
ipv4_address: 172.16.238.4
ports:
- "6379:6379"
volumes:
@@ -144,6 +165,9 @@
env_file:
- .env
restart: on-failure
networks:
backend:
ipv4_address: 172.16.238.3
ports:
- "5432:5432"
volumes:
@@ -151,11 +175,22 @@
- ./sql/schema.sql:/docker-entrypoint-initdb.d/schema.sql:ro
consul:
image: consul:1.15.4
command: agent -dev -client=0.0.0.0
image: hashicorp/consul:latest
command: [
"agent",
"-dev",
"-bootstrap-expect=1",
"-client=0.0.0.0",
"-bind=0.0.0.0",
]
restart: on-failure
networks:
backend:
ipv4_address: 172.16.238.2
ports:
- "8500:8500"
- "8600:8600/udp"
- "8600:8600/tcp"
volumes:
- ./scripts/consul.json:/consul/config/cors.json
@@ -163,3 +198,13 @@ volumes:
db_data:
cache_data:
service_ids:
networks:
backend:
ipam:
driver: default
config:
- subnet: "172.16.238.0/24"
- subnet: "2001:3984:3989::/64"
frontend:
driver: bridge

2
packet-service/src/main.rs
View File

@@ -83,7 +83,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Register service with Consul
let service_id = consul_registration::get_or_generate_service_id(env!("CARGO_PKG_NAME"));
let version = env!("CARGO_PKG_VERSION").to_string();
let tags = vec![version];
let tags = vec![version, "tcp".to_string()];
let mut meta = HashMap::new();
consul_registration::register_service(
&consul_url,

3
scripts/consul.json
View File

@@ -1,4 +1,7 @@
{
"recursors": [
"127.0.0.11"
],
"http_config": {
"response_headers": {
"Access-Control-Allow-Origin": "*"

2
utils/Cargo.toml
View File

@@ -15,3 +15,5 @@ redis = "0.29.1"
deadpool-redis = "0.20.0"
async-trait = "0.1.87"
serde_json = "1.0.140"
hickory-resolver = "0.24.4"
rand = "0.8.5"

6
utils/src/consul_registration.rs
View File

@@ -64,6 +64,12 @@ pub async fn register_service(
"port": service_port,
"tags": tags,
"meta": meta,
"tagged_addresses": {
"lan": {
"address": service_address.to_string(),
"port": service_port,
}
},
"check": {
check_protocol: check_address,
"interval": "10s",

1
utils/src/lib.rs
View File

@@ -3,3 +3,4 @@ pub mod null_string;
pub mod redis_cache;
pub mod service_discovery;
pub mod signal_handler;
pub mod multi_service_load_balancer;

135
utils/src/multi_service_load_balancer.rs Normal file
View File

@@ -0,0 +1,135 @@
use std::collections::HashMap;
use std::net::SocketAddr;
use std::sync::{Arc, Mutex};
use rand::seq::SliceRandom;
use crate::service_discovery::get_service_endpoints_by_dns;
pub enum LoadBalancingStrategy {
Random,
RoundRobin,
}
// Service identifier
#[derive(Clone, PartialEq, Eq, Hash)]
pub struct ServiceId {
pub name: String,
pub protocol: String,
}
impl ServiceId {
pub fn new(name: &str, protocol: &str) -> Self {
ServiceId {
name: name.to_string(),
protocol: protocol.to_string(),
}
}
}
// Per-service state
struct ServiceState {
endpoints: Vec<SocketAddr>,
current_index: usize,
}
impl ServiceState {
fn new(endpoints: Vec<SocketAddr>) -> Self {
ServiceState {
endpoints,
current_index: 0,
}
}
fn get_endpoint(&mut self, strategy: &LoadBalancingStrategy) -> Option<SocketAddr> {
if self.endpoints.is_empty() {
return None;
}
match strategy {
LoadBalancingStrategy::Random => {
let mut rng = rand::thread_rng();
self.endpoints.choose(&mut rng).copied()
}
LoadBalancingStrategy::RoundRobin => {
let endpoint = self.endpoints[self.current_index].clone();
self.current_index = (self.current_index + 1) % self.endpoints.len();
Some(endpoint)
}
}
}
}
pub struct MultiServiceLoadBalancer {
consul_url: String,
strategy: LoadBalancingStrategy,
services: Arc<Mutex<HashMap<ServiceId, ServiceState>>>,
}
impl MultiServiceLoadBalancer {
pub fn new(consul_url: &str, strategy: LoadBalancingStrategy) -> Self {
MultiServiceLoadBalancer {
consul_url: consul_url.to_string(),
strategy,
services: Arc::new(Mutex::new(HashMap::new())),
}
}
pub async fn get_endpoint(
&self,
service_name: &str,
service_protocol: &str,
) -> Result<Option<SocketAddr>, Box<dyn std::error::Error>> {
let service_id = ServiceId::new(service_name, service_protocol);
// Try to get an endpoint from the cache first
{
let mut services = self.services.lock().unwrap();
if let Some(service_state) = services.get_mut(&service_id) {
if let Some(endpoint) = service_state.get_endpoint(&self.strategy) {
return Ok(Some(endpoint));
}
}
}
// If we don't have endpoints or they're all unavailable, refresh them
self.refresh_service_endpoints(service_name, service_protocol).await?;
// Try again after refresh
let mut services = self.services.lock().unwrap();
if let Some(service_state) = services.get_mut(&service_id) {
return Ok(service_state.get_endpoint(&self.strategy));
}
Ok(None)
}
pub async fn refresh_service_endpoints(
&self,
service_name: &str,
service_protocol: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let endpoints = get_service_endpoints_by_dns(
&self.consul_url,
service_protocol,
service_name,
).await?;
let service_id = ServiceId::new(service_name, service_protocol);
let mut services = self.services.lock().unwrap();
services.insert(service_id, ServiceState::new(endpoints));
Ok(())
}
pub async fn refresh_all_services(&self) -> Result<(), Box<dyn std::error::Error>> {
let service_ids: Vec<ServiceId> = {
let services = self.services.lock().unwrap();
services.keys().cloned().collect()
};
for service_id in service_ids {
self.refresh_service_endpoints(&service_id.name, &service_id.protocol).await?;
}
Ok(())
}
}

34
utils/src/service_discovery.rs
View File

@@ -1,5 +1,33 @@
use serde::Deserialize;
use hickory_resolver::config::*;
use hickory_resolver::{Resolver, TokioAsyncResolver};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::net::SocketAddr;
use std::str::FromStr;
use tokio::runtime::Runtime;
use tracing::log::debug;
pub async fn get_service_endpoints_by_dns(consul_url: &str, service_protocol: &str, service_name: &str) -> Result<Vec<SocketAddr>, Box<dyn std::error::Error>> {
let mut rc = ResolverConfig::new();
let url = consul_url.parse()?;
rc.add_name_server(NameServerConfig::new(url, Protocol::Tcp));
let resolver = TokioAsyncResolver::tokio(rc, ResolverOpts::default());
let srv_name = format!("_{}._{}.service.consul", service_name, service_protocol);
let srv_record = resolver.srv_lookup(&srv_name).await?;
let mut endpoints = Vec::new();
for record in srv_record {
let hostname = record.target();
let lookup_responses = resolver.lookup_ip(hostname.to_string()).await?;
for response in lookup_responses {
endpoints.push(SocketAddr::from_str(&format!("{}:{}", &response.to_string(), record.port()))?);
}
}
Ok(endpoints)
}
#[derive(Debug, Deserialize)]
pub struct ServiceNode {
@@ -24,7 +52,7 @@ pub async fn get_service_address(
service_name,
response.status()
)
.into());
.into());
}
// Deserialize the response into a Vec<ServiceNode>
@@ -52,7 +80,7 @@ async fn get_services_with_tag(
service_name,
response.status()
)
.into());
.into());
}
// Deserialize the response into a Vec<ServiceNode>

1
world-service/Cargo.toml
View File

@@ -13,6 +13,7 @@ tracing-subscriber = "0.3.18"
tonic = "0.12.3"
prost = "0.13.4"
warp = "0.3.7"
tonic-health = "0.12.3"
[build-dependencies]
tonic-build = "0.12.3"

16
world-service/src/main.rs
View File

@@ -2,9 +2,9 @@ use dotenv::dotenv;
use std::collections::HashMap;
use std::env;
use std::str::FromStr;
use tracing::Level;
use tracing::{debug, Level};
use utils::consul_registration;
use utils::service_discovery::get_service_address;
use utils::service_discovery::{get_service_address, get_service_endpoints_by_dns};
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -21,15 +21,21 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let port = env::var("WORLD_SERVICE_PORT").unwrap_or_else(|_| "50054".to_string());
let health_port = env::var("HEALTH_CHECK_PORT").unwrap_or_else(|_| "8084".to_string());
let consul_url = env::var("CONSUL_URL").unwrap_or_else(|_| "http://127.0.0.1:8500".to_string());
// let consul_url = env::var("CONSUL_URL").unwrap_or_else(|_| "http://127.0.0.1:8500".to_string());
let consul_address = env::var("CONSUL_ADDRESS").unwrap_or_else(|_| "127.0.0.1".to_string());
let consul_port = env::var("CONSUL_PORT").unwrap_or_else(|_| "8500".to_string());
let consul_dns_port = env::var("CONSUL_DNS_PORT").unwrap_or_else(|_| "8600".to_string());
let consul_url = format!("http://{}:{}", consul_address, consul_port);
let service_name = env::var("SERVICE_NAME").unwrap_or_else(|_| "world-service".to_string());
let service_address =
env::var("WORLD_SERVICE_ADDR").unwrap_or_else(|_| "127.0.0.1".to_string());
let service_port = port.clone();
let health_check_url = format!("http://{}:{}/health", service_address, health_port);
let health_check_endpoint_addr = format!("{}:{}", service_address, health_port);
let db_nodes = get_service_address(&consul_url, "database-service").await?;
let temp_db_nodes = get_service_endpoints_by_dns(format!("{}:{}", consul_address, consul_dns_port).as_str(), "grpc", "database-service").await?;
debug!("{:?}", temp_db_nodes);
// Register service with Consul
let service_id = consul_registration::get_or_generate_service_id(env!("CARGO_PKG_NAME"));
let version = env!("CARGO_PKG_VERSION").to_string();
@@ -47,7 +53,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Some("http"),
Some(&health_check_url),
)
.await?;
.await?;
// Start health-check endpoint
consul_registration::start_health_check(addr.as_str()).await?;