Wiederholte Aufrufe bei Python subprocess.run (['CP', SRC, DST]) verursacht den Speicherfehler unter Linux aus dem Speic ⇐ Linux

[Anonymous]

Ich habe ein relativ einfaches Python -Skript, mit dem zwei Medienspeicher über eine USB -Festplatte synchronisiert werden. Es überprüft die auf einem Remote -Slave -Server vorhandenen Dateien über SSH und vergleicht die Liste mit den auf dem lokalen (Master-) Server vorhandenen Dateien. Die Dateien, die sich auf dem Master -Server befinden, aber auf dem Remote -Slave -Server fehlen, werden auf den USB -HDD kopiert. Ich füge dann den USB -HDD physisch an den Slave -Server hinzu und kopiere die fehlenden Dateien (über ein anderes Skript) an den Slave -Server -HDD. Es wird auf einem PI 4 mit 8 GB RAM ausgeführt und das System zeigt 7,4 GB kostenlos, wenn das Skript gestartet wird. Hier ist das Python-Skript: < /p>
#!/usr/bin/env python3

import os
import subprocess
import json
from datetime import datetime
import argparse
import hashlib

class StorageSync:
def __init__(self, master_path, slave_host, slave_path, usb_path):
self.master_path = os.path.normpath(master_path)
self.slave_host = slave_host
self.slave_path = os.path.normpath(slave_path)
self.usb_path = os.path.normpath(usb_path)

# Create a unique directory name based on the paths
path_hash = hashlib.md5(f"{master_path}:{slave_path}".encode()).hexdigest()[:8]
self.sync_dir = os.path.join(self.usb_path, f"sync_{path_hash}")

def get_file_list(self, path, is_remote=False):
"""Get list of files with their sizes"""
if is_remote:
# Use find with null terminators and stat to handle spaces correctly
cmd = f'ssh {self.slave_host} "find {path} -type f -print0 | xargs -0 stat -c \'%s %n\'"'
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
files = {}
for line in result.stdout.splitlines():
if not line.strip():
continue
# Split on first space only
size_str, filepath = line.split(' ', 1)
try:
size = int(size_str)
rel_path = os.path.relpath(filepath, path)
files[rel_path] = size
except ValueError:
print(f"Warning: Could not parse size from line: {line}")
else:
# Get local file list, including hidden files
files = {}
for root, _, filenames in os.walk(path):
for filename in filenames:
filepath = os.path.join(root, filename)
rel_path = os.path.relpath(filepath, path)
files[rel_path] = os.path.getsize(filepath)
return files

def compare_storage(self):
"""Compare master and slave storage"""
print("Getting master file list...")
master_files = self.get_file_list(self.master_path)
print(f"Found {len(master_files)} files on master")

print("Getting slave file list...")
slave_files = self.get_file_list(self.slave_path, is_remote=True)
print(f"Found {len(slave_files)} files on slave")

to_copy = {}
to_delete = []
size_mismatches = 0

# Find files to copy (new or different size)
for file_path, size in master_files.items():
if file_path not in slave_files:
to_copy[file_path] = size
elif slave_files[file_path] != size:
to_copy[file_path] = size
size_mismatches += 1

# Find files to delete (exists in slave but not in master)
for file_path in slave_files:
if file_path not in master_files:
to_delete.append(file_path)

print(f"Found {size_mismatches} files with size mismatches")

return to_copy, to_delete

def copy_to_usb(self, to_copy):
"""Copy files to USB drive"""
# Create sync directory if it doesn't exist
os.makedirs(self.sync_dir, exist_ok=True)

total_files = len(to_copy)
for i, (rel_path, size) in enumerate(to_copy.items(), 1):
src = os.path.join(self.master_path, rel_path)
dst = os.path.join(self.sync_dir, rel_path)
os.makedirs(os.path.dirname(dst), exist_ok=True)
print(f"Copying file {i}/{total_files}: {rel_path}")
subprocess.run(['cp', src, dst])
print(f"Finished copying file {i}/{total_files}: {rel_path}")

def generate_delete_script(self, to_delete):
"""Generate a script to delete files on slave"""
# Create sync directory if it doesn't exist
os.makedirs(self.sync_dir, exist_ok=True)

script_path = os.path.join(self.sync_dir, 'delete_files.sh')
with open(script_path, 'w') as f:
f.write('#!/bin/bash\n')
f.write(f'# Delete script for {self.master_path} -> {self.slave_path}\n')
f.write(f'# Generated on {datetime.now().isoformat()}\n\n')
for file_path in to_delete:
f.write(f'rm "{os.path.join(self.slave_path, file_path)}"\n')
os.chmod(script_path, 0o755)

def main():
# Get the directory where the script is located
script_dir = os.path.dirname(os.path.abspath(__file__))

parser = argparse.ArgumentParser(description='Sync storage between master and slave servers')
parser.add_argument('--master-path', required=True, help='Path to master storage')
parser.add_argument('--slave-host', required=True, help='Slave server SSH host')
parser.add_argument('--slave-path', required=True, help='Path to slave storage')
parser.add_argument('--usb-path', default=script_dir, help='Path to USB drive (defaults to script directory)')

args = parser.parse_args()

syncer = StorageSync(args.master_path, args.slave_host, args.slave_path, args.usb_path)

print("Comparing storage...")
to_copy, to_delete = syncer.compare_storage()

# Save comparison results
results = {
'timestamp': datetime.now().isoformat(),
'master_path': args.master_path,
'slave_path': args.slave_path,
'to_copy': to_copy,
'to_delete': to_delete
}

# Create sync directory if it doesn't exist
os.makedirs(syncer.sync_dir, exist_ok=True)

with open(os.path.join(syncer.sync_dir, 'sync_plan.json'), 'w') as f:
json.dump(results, f, indent=2)

print(f"Files to copy: {len(to_copy)}")
print(f"Files to delete: {len(to_delete)}")
print(f"Sync directory: {syncer.sync_dir}")

if to_copy:
print("Copying files to USB...")
syncer.copy_to_usb(to_copy)

if to_delete:
print("Generating delete script...")
syncer.generate_delete_script(to_delete)

print("Done!")

if __name__ == '__main__':
main()
< /code>
Das Python-Skript wird 4 Mal aus einem Bash-Skript aufgerufen, um 4 verschiedene Verzeichnisse zu synchronisieren: < /p>
#! /bin/bash
echo "###### Syncing myuser/downloads ######"
./storage_sync.py --master-path /home/myuser/downloads --slave-host mediauser@slave-server --slave-path /home/mediauser/downloads

echo "###### Syncing videos ######"
./storage_sync.py --master-path /home/myuser/videos --slave-host mediauser@slave-server --slave-path /home/mediauser/videos

echo "###### Syncing keepers ######"
./storage_sync.py --master-path /home/myuser/keepers --slave-host mediauser@slave-server --slave-path /home/mediauser/keepers

echo "###### Syncing dlna content ######"
./storage_sync.py --master-path /dlna/videos --slave-host mediauser@slave-server --slave-path /home/minidlna/share/videos

< /code>
Kann mir jemand sagen, warum dies der Speicher ausgeht? Ich dachte>