Wiederholte Aufrufe bei Python subprocess.run (['cp', src, dst]) kann den Speicher nicht zuweisen ⇐ Linux

[Anonymous]

Update Ich habe dem Skript eine Speicherprotokollierung hinzugefügt. Dies ist die Ausgabe: < /p>

Code: Select all

Comparing storage...
Getting master file list...
Found 22958 files on master
Getting slave file list...
Found 13745 files on slave
Found 1 files with size mismatches
Files to copy: 9214
Files to delete: 0
Sync directory: /mnt/usbhdd/sync_35607613
Copying files to USB...
Available system memory after copying 1/9214: 7451 MB
Available system memory after copying 2/9214: 7450 MB
Available system memory after copying 3/9214: 7450 MB
.
.
. [lines removed for brevity]
.
.
Available system memory after copying 354/9214: 7413 MB
Available system memory after copying 355/9214: 7445 MB
Available system memory after copying 356/9214: 7446 MB
Available system memory after copying 357/9214: 7446 MB
Traceback (most recent call last):
File "/mnt/usbhdd/./storage_sync.py", line 173, in 
main()
File "/mnt/usbhdd/./storage_sync.py", line 164, in main
syncer.copy_to_usb(to_copy)
File "/mnt/usbhdd/./storage_sync.py", line 107, in copy_to_usb
subprocess.run(['cp', src, dst])
File "/usr/lib/python3.9/subprocess.py", line 505, in run
with Popen(*popenargs, **kwargs) as process:
File "/usr/lib/python3.9/subprocess.py", line 951, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
File "/usr/lib/python3.9/subprocess.py", line 1756, in _execute_child
self.pid = _posixsubprocess.fork_exec(
OSError: [Errno 12] Cannot allocate memory
< /code>
 Die ursprüngliche Frage: < /strong>
Ich habe ein relativ einfaches Python -Skript, mit dem zwei Medienspeicher über ein USB -HDD synchronisiert werden. Es überprüft die auf einem Remote -Slave -Server vorhandenen Dateien über SSH und vergleicht die Liste mit den auf dem lokalen (Master-) Server vorhandenen Dateien. Die Dateien, die sich auf dem Master -Server befinden, aber auf dem Remote -Slave -Server fehlen, werden auf den USB -HDD kopiert.  Ich füge das USB -HDD dann physisch an den Slave -Server hinzu und kopiere die fehlenden Dateien (über ein anderes Skript) an den Slave -Server -HDD.  Es wird auf einem PI 4 mit 8 GB RAM ausgeführt und das System zeigt 7,4 GB kostenlos, wenn das Skript gestartet wird.   Hier ist das Python-Skript: < /p>
#!/usr/bin/env python3

import os
import subprocess
import json
from datetime import datetime
import argparse
import hashlib
import psutil

class StorageSync:
def __init__(self, master_path, slave_host, slave_path, usb_path):
self.master_path = os.path.normpath(master_path)
self.slave_host = slave_host
self.slave_path = os.path.normpath(slave_path)
self.usb_path = os.path.normpath(usb_path)

# Create a unique directory name based on the paths
path_hash = hashlib.md5(f"{master_path}:{slave_path}".encode()).hexdigest()[:8]
self.sync_dir = os.path.join(self.usb_path, f"sync_{path_hash}")

def get_available_memory(self):
"""Get available system memory in MB"""
try:
memory = psutil.virtual_memory()
return memory.available // (1024 * 1024)  # Convert to MB
except Exception as e:
print(f"Warning: Could not get memory info: {e}")
return None

def log_memory_status(self, file_path, file_number, total_files):
"""Log available system memory after copying a file"""
available_mb = self.get_available_memory()
if available_mb is not None:
print(f"Available system memory after copying {file_number}/{total_files}: {available_mb} MB")

def get_file_list(self, path, is_remote=False):
"""Get list of files with their sizes"""
if is_remote:
# Use find with null terminators and stat to handle spaces correctly
cmd = f'ssh {self.slave_host} "find {path} -type f -print0 | xargs -0 stat -c \'%s %n\'"'
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
files = {}
for line in result.stdout.splitlines():
if not line.strip():
continue
# Split on first space only
size_str, filepath = line.split(' ', 1)
try:
size = int(size_str)
rel_path = os.path.relpath(filepath, path)
files[rel_path] = size
except ValueError:
print(f"Warning: Could not parse size from line:  {line}")
else:
# Get local file list, including hidden files
files = {}
for root, _, filenames in os.walk(path):
for filename in filenames:
filepath = os.path.join(root, filename)
rel_path = os.path.relpath(filepath, path)
files[rel_path] = os.path.getsize(filepath)
return files

def compare_storage(self):
"""Compare master and slave storage"""
print("Getting master file list...")
master_files = self.get_file_list(self.master_path)
print(f"Found {len(master_files)} files on master")

print("Getting slave file list...")
slave_files = self.get_file_list(self.slave_path, is_remote=True)
print(f"Found {len(slave_files)} files on slave")

to_copy = {}
to_delete = []
size_mismatches = 0

# Find files to copy (new or different size)
for file_path, size in master_files.items():
if file_path not in slave_files:
to_copy[file_path] = size
elif slave_files[file_path] != size:
to_copy[file_path] = size
size_mismatches += 1

# Find files to delete (exists in slave but not in master)
for file_path in slave_files:
if file_path not in master_files:
to_delete.append(file_path)

print(f"Found {size_mismatches} files with size mismatches")

return to_copy, to_delete

def copy_to_usb(self, to_copy):
"""Copy files to USB drive"""
# Create sync directory if it doesn't exist
os.makedirs(self.sync_dir, exist_ok=True)

total_files = len(to_copy)
for i, (rel_path, size) in enumerate(to_copy.items(), 1):
src = os.path.join(self.master_path, rel_path)
dst = os.path.join(self.sync_dir, rel_path)
os.makedirs(os.path.dirname(dst), exist_ok=True)
#print(f"Copying file {i}/{total_files}: {rel_path}")
subprocess.run(['cp', src, dst])

# Check and log available memory after each file copy
self.log_memory_status(rel_path, i, total_files)

def generate_delete_script(self, to_delete):
"""Generate a script to delete files on slave"""
# Create sync directory if it doesn't exist
os.makedirs(self.sync_dir, exist_ok=True)

script_path = os.path.join(self.sync_dir, 'delete_files.sh')
with open(script_path, 'w') as f:
f.write('#!/bin/bash\n')
f.write(f'# Delete script for {self.master_path} ->  {self.slave_path}\n')
f.write(f'# Generated on {datetime.now().isoformat()}\n\n')
for file_path in to_delete:
f.write(f'rm "{os.path.join(self.slave_path, file_path)}"\n')
os.chmod(script_path, 0o755)

def main():
# Get the directory where the script is located
script_dir = os.path.dirname(os.path.abspath(__file__))

parser = argparse.ArgumentParser(description='Sync storage between master and slave servers')
parser.add_argument('--master-path', required=True, help='Path to master storage')
parser.add_argument('--slave-host', required=True, help='Slave server SSH host')
parser.add_argument('--slave-path', required=True, help='Path to slave storage')
parser.add_argument('--usb-path', default=script_dir, help='Path to USB drive (defaults to script directory)')

args = parser.parse_args()

syncer = StorageSync(args.master_path, args.slave_host, args.slave_path, args.usb_path)

print("Comparing storage...")
to_copy, to_delete = syncer.compare_storage()

# Save comparison results
results = {
'timestamp': datetime.now().isoformat(),
'master_path': args.master_path,
'slave_path': args.slave_path,
'to_copy': to_copy,
'to_delete': to_delete
}

# Create sync directory if it doesn't exist
os.makedirs(syncer.sync_dir, exist_ok=True)

with open(os.path.join(syncer.sync_dir, 'sync_plan.json'), 'w') as f:
json.dump(results, f, indent=2)

print(f"Files to copy: {len(to_copy)}")
print(f"Files to delete: {len(to_delete)}")
print(f"Sync directory: {syncer.sync_dir}")

if to_copy:
print("Copying files to USB...")
syncer.copy_to_usb(to_copy)

if to_delete:
print("Generating delete script...")
syncer.generate_delete_script(to_delete)

print("Done!")

if __name__ == '__main__':
main()

Kann mir jemand sagen, warum dies nicht in der Lage ist, Speicher zuzuweisen?