Herunterladen von Azure Blob in C# vs. Rust ⇐ C#

[Anonymous]

Ich habe einen Blob-Speicher mit 1000 JSON-Dateien und möchte Code von C# nach Rust portieren. Ich war überrascht, dass mein Rust 3-6 mal langsamer war. C# benötigte durchweg 2 Sekunden und Rust irgendwo zwischen 6 und 12 Sekunden, natürlich auf derselben Maschine. Beide wurden natürlich bis zur Veröffentlichung kompiliert.
Liegt das daran, dass C# auch Microsoft ist und nur ein optimierteres Paket ist, oder mache ich es falsch?
C#-Funktion:

Code: Select all

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Azure.Identity;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Newtonsoft.Json;
using OToo.Models;
using DotNetEnv;
using Azure;
using System.Collections.Concurrent;

namespace OToo.Services
{
public class AzureBlobService
{
private readonly string? _clientId;
private readonly string? _tenantId;
private readonly string? _clientSecret;
private readonly string? _accountName;
private readonly string? _containerName;

public AzureBlobService()
{
Env.Load();

_clientId = Environment.GetEnvironmentVariable("AZURE_CLIENT_ID");
_tenantId = Environment.GetEnvironmentVariable("AZURE_TENANT_ID");
_clientSecret = Environment.GetEnvironmentVariable("AZURE_CLIENT_SECRET");
_accountName = Environment.GetEnvironmentVariable("AZURE_ACCOUNT_NAME");
_containerName = Environment.GetEnvironmentVariable("AZURE_CONTAINER_NAME");

if (string.IsNullOrEmpty(_clientId) || string.IsNullOrEmpty(_tenantId) ||
string.IsNullOrEmpty(_clientSecret) || string.IsNullOrEmpty(_accountName) ||
string.IsNullOrEmpty(_containerName))
{
throw new InvalidOperationException("Missing required Azure credentials in [url=viewtopic.php?t=25360]environment[/url] variables.");
}
}

private BlobContainerClient GetBlobContainerClient()
{
var credential = new ClientSecretCredential(_tenantId, _clientId, _clientSecret);
var blobServiceClient = new BlobServiceClient(new Uri($"https://{_accountName}.blob.core.windows.net"), credential);
return blobServiceClient.GetBlobContainerClient(_containerName);
}

public async Task LoadTradesFromFolder(string? prefix)
{
if (string.IsNullOrEmpty(prefix))
{
return new List();
}

var trades = new ConcurrentBag();
var containerClient = GetBlobContainerClient();
var blobItems = new List();

await foreach (var blobItem in containerClient.GetBlobsAsync(prefix: prefix))
{
blobItems.Add(blobItem);
}

await Parallel.ForEachAsync(blobItems, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, async (blobItem, _) =>
{
var blobClient = containerClient.GetBlobClient(blobItem.Name);

try
{
var response = await blobClient.DownloadContentAsync();
var jsonString = Encoding.UTF8.GetString(response.Value.Content.ToArray());
var trade = JsonConvert.DeserializeObject(jsonString);

if (trade != null)
{
trades.Add(trade);
}
}
catch (JsonException ex)
{
Console.WriteLine($"Failed to parse JSON from {blobItem.Name}: {ex.Message}");
}
catch (RequestFailedException ex)
{
Console.WriteLine($"Failed to download {blobItem.Name}: {ex.Message}");
}
});

return trades.ToList();
}

public async Task FetchAllTrades(List prefixes)
{
var allTrades = new List();
var tradeTasks = prefixes.Where(p => !string.IsNullOrEmpty(p))
.Select(prefix =>  LoadTradesFromFolder(prefix!));
var tradeLists = await Task.WhenAll(tradeTasks);

foreach (var tradeList in tradeLists)
{
allTrades.AddRange(tradeList);
}

return allTrades;
}
}
}

Und hier der Rust-Code (zwei Versuche, führen jedoch ähnlich aus):

Code: Select all

use azure_storage::StorageCredentials;
use azure_storage_blobs::prelude::*;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use futures::stream::{StreamExt, FuturesUnordered};
use std::env;

use crate::TradeData;

#[derive(Serialize)]
struct TokenRequest Result {
let start = std::time::Instant::now();
println!("[LOG] [Azure OPTIMIZED] Starting fetch_trades_from_azure_optimized");

let tenant_id = env::var("AZURE_TENANT_ID").map_err(|e| e.to_string())?;
let client_id = env::var("AZURE_CLIENT_ID").map_err(|e| e.to_string())?;
let client_secret = env::var("AZURE_CLIENT_SECRET").map_err(|e| e.to_string())?;
let storage_account = env::var("AZURE_ACCOUNT_NAME").map_err(|e| e.to_string())?;
let container_name = env::var("AZURE_CONTAINER_NAME").map_err(|e| e.to_string())?;
let prefix_opt = env::var("AZURE_PREFIX_OPT").map_err(|e| e.to_string())?;
let prefix_fut = env::var("AZURE_PREFIX_FUT").map_err(|e| e.to_string())?;

let auth_start = std::time::Instant::now();
println!("[LOG] [Azure OPTIMIZED] Authentication step started");
let token = get_oauth_token(&tenant_id, &client_id, &client_secret).await.map_err(|e| e.to_string())?;
println!("[LOG] [Azure OPTIMIZED] Authentication took {:.2} seconds", auth_start.elapsed().as_secs_f32());

let credentials = StorageCredentials::bearer_token(token);
let blob_service = BlobServiceClient::new(storage_account, credentials);
let container_client = blob_service.container_client(container_name);

let list_start = std::time::Instant::now();
println!("[LOG] [Azure OPTIMIZED] List+Download (parallel prefixes) started");

// Process both prefixes in parallel (same as baseline but concurrent)
let (trades_opt, trades_fut) = futures::join!(
process_blobs(&container_client, prefix_opt),
process_blobs(&container_client, prefix_fut)
);

println!("[LOG] [Azure OPTIMIZED] List+Download took {:.2} seconds", list_start.elapsed().as_secs_f32());

let trades: Vec = trades_opt.into_iter().chain(trades_fut).map(|(_name, trade)| trade).collect();

println!("[LOG] [Azure OPTIMIZED] Total: {} trades in {:.2} seconds", trades.len(), start.elapsed().as_secs_f32());
Ok(trades)
}