Warum weichen projizierte Tastoint in der Pyrender -Kamera -Simulation leicht von der Bodenwahrheit ab? ⇐ Python

[Anonymous]

Um eine Kamera -Kalibrierungsbibliothek zu testen, schrieb ich ein Python -Skript mit Pyrender, in dem ich eine Lochkamera eingerichte, um eine Gitterkreisplatine zu erfassen. Der Code ist verfügbar unter: https://github.com/tanjoe/camsim.

, wenn das Running -App -APT -APT -AP -APT aufträgt. Pixelpositionen in einer JSON-Datei.import os
import datetime
import cv2
import pyrender
import trimesh
import json
import numpy as np
import pyglet
from pyrender.constants import GLTF

def projectWorldToImage(
camera: pyrender.IntrinsicsCamera, camera_pose: np.ndarray, world_point: np.ndarray
) -> tuple[float, float]:
# Transform the point to camera coordinates
point_3d_camera = np.linalg.inv(camera_pose) @ world_point
point_3d_camera = point_3d_camera[:3]
X, Y, Z = point_3d_camera

# Project the 3D point to 2D image coordinates
x = camera.cx + (camera.fx * X) / -Z
# Flip the Y-axis to follow OpenCV convention
y = camera.cy - (camera.fy * Y) / -Z

return (x, y)

class MyViewer(pyrender.Viewer):
def __init__(
self,
scene: pyrender.Scene,
camera: pyrender.IntrinsicsCamera,
interested_points: list[np.ndarray],
viewport_size: tuple[int, int],
render_flags=None,
viewer_flags=None,
registered_keys=None,
run_in_thread=False,
**kwargs,
):
self.camera = camera
self.interested_points = interested_points
super().__init__(
scene,
viewport_size,
render_flags,
viewer_flags,
registered_keys,
run_in_thread,
**kwargs,
)

def on_key_press(self, symbol, modifiers):
if symbol == pyglet.window.key.ENTER:
timestamp = datetime.datetime.now().strftime("%H%M%S")
pyglet.image.get_buffer_manager().get_color_buffer().save(
f"output/{timestamp}.png"
)

# Get the current camera pose
camera_pose = self.scene.get_pose(self.scene.main_camera_node)
print(f"Camera pose: {camera_pose}")
np.savetxt(f"output/{timestamp}-camera_pose.txt", camera_pose)

loc_truth = []
for point in self.interested_points:
loc_truth.append(projectWorldToImage(self.camera, camera_pose, point))
with open(f"output/{timestamp}-loc_truth.json", "w") as truth_file:
json.dump(loc_truth, truth_file, indent=4)

return super().on_key_press(symbol, modifiers)

def createBoard(image_path: str) -> pyrender.Mesh:
# Create a texture from the image
image = cv2.imread(image_path)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
sampler = pyrender.Sampler(
magFilter=GLTF.NEAREST,
minFilter=None,
wrapS=GLTF.CLAMP_TO_EDGE,
wrapT=GLTF.CLAMP_TO_EDGE,
)
texture = pyrender.Texture(
sampler=sampler,
source=image,
source_channels="RGB",
width=image.shape[1],
height=image.shape[0],
)

# Create a material using the texture
material = pyrender.MetallicRoughnessMaterial(
baseColorTexture=texture,
emissiveTexture=texture,
emissiveFactor=[0.9, 0.9, 0.9],
doubleSided=False,
smooth=False,
)

# Create a plane mesh with UV coordinates
# Compute vertices based on image resolution to keep the aspect ratio
half_w = image.shape[1] / 1000.0 / 2.0
half_h = image.shape[0] / 1000.0 / 2.0
vertices = np.array(
[
[-half_w, -half_h, 0], # Bottom-left
[half_w, -half_h, 0], # Bottom-right
[-half_w, half_h, 0], # Top-left
[half_w, half_h, 0], # Top-right
],
dtype=np.float64,
)

faces = np.array(
[
[0, 1, 2], # First triangle
[1, 3, 2], # Second triangle
],
dtype=np.uint32,
)

# Define UV coordinates for texture mapping
uv_coords = np.array(
[
[0, 0], # Bottom-left
[1, 0], # Bottom-right
[0, 1], # Top-left
[1, 1], # Top-right
],
dtype=np.float64,
)

# Create a Trimesh object with vertices, faces, and UV coordinates
plane = trimesh.Trimesh(
vertices=vertices,
faces=faces,
visual=trimesh.visual.TextureVisuals(uv=uv_coords, image=image),
)

# Create a Pyrender mesh from the Trimesh object
plane_mesh = pyrender.Mesh.from_trimesh(plane, material=material, smooth=False)
return plane_mesh

def computeCircleCoordinates(json_path: str, board_mesh: pyrender.Mesh) -> np.ndarray:
coordinates: list[list[float]] = []
with open(json_path, "r") as content:
board_info = json.load(content)
image_size = board_info["image_size"]
centers = board_info["centers"]

width_ratio = board_mesh.extents[0] / image_size[0]
height_ratio = board_mesh.extents[1] / image_size[1]
start_x = board_mesh.bounds[0][0]
start_y = board_mesh.bounds[0][1]

for c in centers:
# Flip y to make it follows OpenGL convention (+Y should be upward)
c[1] = image_size[1] - c[1]
x = start_x + c[0] * width_ratio
y = start_y + c[1] * height_ratio
coordinates.append([x, y, 0])
return np.array(coordinates)

def main() -> None:
plane_mesh = createBoard("resource/board.png")
plane_pose = np.eye(4)
plane_pose[2, 3] = -10
plane_node = pyrender.Node(mesh=plane_mesh, matrix=plane_pose)

centers = computeCircleCoordinates("resource/board.json", plane_mesh)
transformed_centers = []
for center in centers:
center_homo = np.append(center, 1)
transformed_centers.append(plane_pose @ center_homo)

# Create a camera at the origin looking down the z-axis
view_width = 1920
view_height = 1080
camera = pyrender.IntrinsicsCamera(
fx=view_width, fy=view_width, cx=(view_width / 2), cy=(view_height / 2)
)
camera_pose = np.eye(4)
camera_node = pyrender.Node(camera=camera, matrix=camera_pose)

# Create a scene
scene = pyrender.Scene(bg_color=[0, 0, 0, 1])
scene.add_node(plane_node)
scene.add_node(camera_node)

# Render the scene using the interactive viewer
os.makedirs("./output", exist_ok=True)
MyViewer(
scene,
camera=camera,
interested_points=transformed_centers,
viewport_size=(view_width, view_height),
)

if __name__ == "__main__":
main()
< /code>
Um zu überprüfen, ob die berechneten Wahrheitswerte korrekt sind, habe ich mehrere Bilder mit der Kamera aufgenommen, die direkt zur Platine gerichtet ist, ohne Rotation. Dann habe ich ein anderes Skript geschrieben, um das erfasste Bild zu laden und alle Keypoints darauf zu überlösen.import os
import cv2
import json
import matplotlib.pyplot as plt
import typer
from pathlib import Path

def selectImageFromDir(img_dir: Path) -> Path:
if not img_dir.exists():
print(f"Error: '{img_dir}' directory not found.")
raise typer.Exit(code=1)

image_files = [f for f in os.listdir(img_dir) if f.endswith(".png")]
if not image_files:
print(f"Error: No .png images found in the '{img_dir}' directory.")
raise typer.Exit(code=1)

print("Available images:")
for i, file in enumerate(image_files):
print(f"{i + 1}. {file}")

while True:
try:
choice = int(input("Enter the number of the image to process: ")) - 1
if 0
Ich erwarte, dass alle Keypoints fast perfekt mit den Kreisen im Bild übereinstimmen. Es gibt jedoch eine kleine Abweichung zwischen ihnen. Das Problem wird deutlicher, wenn die Kamera näher an die Platine vergrenzt ist. />
Ich verstehe, dass numerische Fehler existieren, wenn die Wahrheitswerte berechnet werden. beobachtet.
Wo hätte ich einen Fehler gemacht haben?