Code zum Extrahieren des Kopfes (Gesicht und Haare) aus einem Bild ⇐ IOS

[Anonymous]

Ich habe maskPerson() und dann maskFace() basierend auf VNGeneratePersonSegmentationRequest() bzw. VNFaceObservation implementiert, wie im folgenden Beispiel, aber manchmal sind Teile der Schultern enthalten und die Genauigkeit ist schlecht.
Gibt es eine bessere Möglichkeit oder API, dies vollständig auf einem iOS-Gerät zu erreichen?

Code: Select all

func maskPerson(from image: UIImage) -> UIImage? {
guard let cgImage = image.cgImage else { return nil }

let request = VNGeneratePersonSegmentationRequest()
request.qualityLevel = .accurate
request.outputPixelFormat = kCVPixelFormatType_OneComponent8

let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
do {
try handler.perform([request])
guard let maskBuffer = request.results?.first?.pixelBuffer else { return nil }

let originalCI = CIImage(cgImage: cgImage)
let maskImage = CIImage(cvPixelBuffer: maskBuffer)
let maskScaleX = originalCI.extent.width / maskImage.extent.width
let maskScaleY = originalCI.extent.height / maskImage.extent.height
let scaledMask = maskImage.transformed(
by: __CGAffineTransformMake(maskScaleX, 0, 0, maskScaleY, 0, 0)
)

let maskedCI = originalCI
.applyingFilter("CIBlendWithMask", parameters: [
"inputMaskImage": scaledMask
])

let context = CIContext()
if let outputCG = context.createCGImage(maskedCI, from: maskedCI.extent) {
return UIImage(cgImage: outputCG)
}
} catch {
print("Segmentation failed: \(error)")
}
return nil
}

Code: Select all

func maskFace(in image: UIImage, faceObservation: VNFaceObservation) -> UIImage? {
let size = image.size // points
UIGraphicsBeginImageContextWithOptions(size, false, image.scale)
guard let context = UIGraphicsGetCurrentContext() else {
UIGraphicsEndImageContext()
return image
}

context.clear(CGRect(origin: .zero, size: size))

let imageWidth = size.width
let imageHeight = size.height
let bbox = faceObservation.boundingBox // normalized (0..1)

// normalized (boundingBox内) -> image coordinate (points)
func convertNormalizedPoint(_ p: CGPoint) -> CGPoint {
// x: boundingBox.origin.x + p.x * boundingBox.width
let x = (bbox.origin.x + p.x * bbox.size.width) * imageWidth
// y: Vision の normalized は bottom-origin、UIKit は top-origin なので反転
let y = (1.0 - (bbox.origin.y + p.y * bbox.size.height)) * imageHeight
return CGPoint(x: x, y: y)
}

if let landmarks = faceObservation.landmarks,
let faceContour = landmarks.faceContour,
faceContour.pointCount >= 3 {

let pts = faceContour.normalizedPoints.map { convertNormalizedPoint($0) }

let path = UIBezierPath()
path.move(to: pts[0])
for p in pts.dropFirst() {
path.addLine(to: p)
}
path.close()

var mouthTopY: CGFloat?
if let outerLips = landmarks.outerLips, outerLips.pointCount > 0 {
let lipPoints = outerLips.normalizedPoints.map { convertNormalizedPoint($0) }
mouthTopY = lipPoints.map { $0.y }.min()
}

let cutY = mouthTopY ?? (size.height / 3)

context.saveGState()

image.draw(in: CGRect(origin: .zero, size: size))

context.addRect(CGRect(origin: .zero, size: size))
context.addPath(path.cgPath)
context.clip(using: .evenOdd)

let transparentArea = CGRect(x: 0, y: cutY, width: size.width, height: size.height - cutY)
context.clear(transparentArea)

context.restoreGState()

let maskedImage = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
return maskedImage
} else {
return image
}
}