/*
* Copyright (C) 2011 Jason von Nieda <jason@vonnieda.org>
*
* This file is part of OpenPnP.
*
* OpenPnP is free software: you can redistribute it and/or modify it under the terms of the GNU
* General Public License as published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* OpenPnP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
* the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with OpenPnP. If not, see
* <http://www.gnu.org/licenses/>.
*
* For more information about OpenPnP visit http://openpnp.org
*/
package org.openpnp.machine.reference;
import java.awt.event.ActionEvent;
import java.awt.image.BufferedImage;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import javax.swing.AbstractAction;
import javax.swing.Action;
import javax.swing.JOptionPane;
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.Point;
import org.opencv.core.Rect;
import org.opencv.core.RotatedRect;
import org.opencv.imgproc.Imgproc;
import org.openpnp.ConfigurationListener;
import org.openpnp.gui.MainFrame;
import org.openpnp.gui.support.Icons;
import org.openpnp.gui.support.MessageBoxes;
import org.openpnp.gui.support.PropertySheetWizardAdapter;
import org.openpnp.gui.wizards.CameraConfigurationWizard;
import org.openpnp.model.Configuration;
import org.openpnp.model.Length;
import org.openpnp.model.LengthUnit;
import org.openpnp.model.Location;
import org.openpnp.spi.base.AbstractCamera;
import org.openpnp.util.OpenCvUtils;
import org.openpnp.vision.LensCalibration;
import org.openpnp.vision.LensCalibration.LensModel;
import org.openpnp.vision.LensCalibration.Pattern;
import org.pmw.tinylog.Logger;
import org.simpleframework.xml.Attribute;
import org.simpleframework.xml.Element;
import org.simpleframework.xml.core.Commit;
import org.simpleframework.xml.core.Persist;
public abstract class ReferenceCamera extends AbstractCamera implements ReferenceHeadMountable {
static {
nu.pattern.OpenCV.loadShared();
System.loadLibrary(org.opencv.core.Core.NATIVE_LIBRARY_NAME);
}
@Element(required = false)
private Location headOffsets = new Location(LengthUnit.Millimeters);
@Attribute(required = false)
protected double rotation = 0;
@Attribute(required = false)
protected boolean flipX = false;
@Attribute(required = false)
protected boolean flipY = false;
@Element(required = false)
protected Length safeZ = new Length(0, LengthUnit.Millimeters);
@Attribute(required = false)
protected int offsetX = 0;
@Attribute(required = false)
protected int offsetY = 0;
@Attribute(required = false)
protected int cropWidth = 0;
@Attribute(required = false)
protected int cropHeight = 0;
@Element(required = false)
private LensCalibrationParams calibration = new LensCalibrationParams();
private boolean calibrating;
private CalibrationCallback calibrationCallback;
private int calibrationCountGoal = 25;
private Mat undistortionMap1;
private Mat undistortionMap2;
private LensCalibration lensCalibration;
public ReferenceCamera() {
}
@Override
public BufferedImage capture() {
try {
Map<String, Object> globals = new HashMap<>();
globals.put("camera", this);
Configuration.get().getScripting().on("Camera.BeforeCapture", globals);
}
catch (Exception e) {
Logger.warn(e);
}
BufferedImage image;
while ((image = internalCapture()) == null) {
System.out.println("got a null");
}
try {
Map<String, Object> globals = new HashMap<>();
globals.put("camera", this);
Configuration.get().getScripting().on("Camera.AfterCapture", globals);
}
catch (Exception e) {
Logger.warn(e);
}
return image;
}
protected abstract BufferedImage internalCapture();
@Override
public int getWidth() {
if (width == null) {
BufferedImage image = capture();
width = image.getWidth();
height = image.getHeight();
}
return width;
}
@Override
public int getHeight() {
if (width == null) {
BufferedImage image = capture();
width = image.getWidth();
height = image.getHeight();
}
return height;
}
@Override
public Location getHeadOffsets() {
return headOffsets;
}
@Override
public void setHeadOffsets(Location headOffsets) {
this.headOffsets = headOffsets;
}
@Override
public void moveTo(Location location, double speed) throws Exception {
Logger.debug("moveTo({}, {})", location, speed);
getDriver().moveTo(this, location, speed);
getMachine().fireMachineHeadActivity(head);
}
@Override
public void moveToSafeZ(double speed) throws Exception {
Logger.debug("{}.moveToSafeZ({})", getName(), speed);
Length safeZ = this.safeZ.convertToUnits(getLocation().getUnits());
Location l = new Location(getLocation().getUnits(), Double.NaN, Double.NaN,
safeZ.getValue(), Double.NaN);
getDriver().moveTo(this, l, speed);
getMachine().fireMachineHeadActivity(head);
}
public double getRotation() {
return rotation;
}
public void setRotation(double rotation) {
this.rotation = rotation;
}
public boolean isFlipX() {
return flipX;
}
public void setFlipX(boolean flipX) {
this.flipX = flipX;
}
public boolean isFlipY() {
return flipY;
}
public void setFlipY(boolean flipY) {
this.flipY = flipY;
}
public int getOffsetX() {
return offsetX;
}
public void setOffsetX(int offsetX) {
this.offsetX = offsetX;
}
public int getOffsetY() {
return offsetY;
}
public void setOffsetY(int offsetY) {
this.offsetY = offsetY;
}
public int getCropWidth() {
return cropWidth;
}
public void setCropWidth(int cropWidth) {
this.cropWidth = cropWidth;
}
public int getCropHeight() {
return cropHeight;
}
public void setCropHeight(int cropHeight) {
this.cropHeight = cropHeight;
}
protected BufferedImage transformImage(BufferedImage image) {
Mat mat = OpenCvUtils.toMat(image);
mat = crop(mat);
mat = calibrate(mat);
mat = undistort(mat);
// apply affine transformations
mat = rotate(mat, rotation);
mat = offset(mat, offsetX, offsetY);
if (flipX || flipY) {
int flipCode;
if (flipX && flipY) {
flipCode = -1;
}
else {
flipCode = flipX ? 0 : 1;
}
Core.flip(mat, mat, flipCode);
}
image = OpenCvUtils.toBufferedImage(mat);
mat.release();
return image;
}
private Mat crop(Mat mat) {
if (cropWidth != 0 || cropHeight != 0) {
int cw = (cropWidth != 0) ? cropWidth : (int) mat.size().width;
int ch = (cropHeight != 0) ? cropHeight : (int) mat.size().height;
Rect roi = new Rect(
(int) ((mat.size().width / 2) - (cw / 2)),
(int) ((mat.size().height / 2) - (ch / 2)),
cw,
ch);
Mat tmp = new Mat(mat, roi);
tmp.copyTo(mat);
tmp.release();
}
return mat;
}
private Mat rotate(Mat mat, double rotation) {
if (rotation == 0D) {
return mat;
}
// See:
// http://stackoverflow.com/questions/22041699/rotate-an-image-without-cropping-in-opencv-in-c
Point center = new Point(mat.width() / 2D, mat.height() / 2D);
Mat mapMatrix = Imgproc.getRotationMatrix2D(center, rotation, 1.0);
// determine bounding rectangle
Rect bbox = new RotatedRect(center, mat.size(), rotation).boundingRect();
// adjust transformation matrix
double[] cx = mapMatrix.get(0, 2);
double[] cy = mapMatrix.get(1, 2);
cx[0] += bbox.width / 2D - center.x;
cy[0] += bbox.height / 2D - center.y;
mapMatrix.put(0, 2, cx);
mapMatrix.put(1, 2, cy);
Mat dst = new Mat(bbox.width, bbox.height, mat.type());
Imgproc.warpAffine(mat, dst, mapMatrix, bbox.size(), Imgproc.INTER_LINEAR);
mat.release();
mapMatrix.release();
return dst;
}
private Mat offset(Mat mat, int offsetX, int offsetY) {
if (offsetX == 0D && offsetY == 0D) {
return mat;
}
Mat mapMatrix = new Mat(2, 3, CvType.CV_32F) {
{
put(0, 0, 1, 0, offsetX);
put(1, 0, 0, 1, offsetY);
}
};
Mat dst = mat.clone();
Imgproc.warpAffine(mat, dst, mapMatrix, mat.size(), Imgproc.INTER_LINEAR);
mat.release();
mapMatrix.release();
return dst;
}
private Mat undistort(Mat mat) {
if (!calibration.isEnabled()) {
return mat;
}
if (undistortionMap1 == null || undistortionMap2 == null) {
undistortionMap1 = new Mat();
undistortionMap2 = new Mat();
Mat rectification = Mat.eye(3, 3, CvType.CV_32F);
Imgproc.initUndistortRectifyMap(calibration.getCameraMatrixMat(),
calibration.getDistortionCoefficientsMat(), rectification,
calibration.getCameraMatrixMat(), mat.size(), CvType.CV_32FC1, undistortionMap1,
undistortionMap2);
rectification.release();
}
Mat dst = mat.clone();
Imgproc.remap(mat, dst, undistortionMap1, undistortionMap2, Imgproc.INTER_LINEAR);
mat.release();
return dst;
}
private Mat calibrate(Mat mat) {
if (!calibrating) {
return mat;
}
int count = lensCalibration.getPatternFoundCount();
Mat appliedMat = lensCalibration.apply(mat);
if (appliedMat == null) {
// nothing was found in the image
return mat;
}
if (count != lensCalibration.getPatternFoundCount()) {
// a new image was counted, so let the caller know
if (lensCalibration.getPatternFoundCount() == calibrationCountGoal) {
calibrationCallback.callback(lensCalibration.getPatternFoundCount(),
calibrationCountGoal, true);
lensCalibration.calibrate();
calibration.setCameraMatrixMat(lensCalibration.getCameraMatrix());
calibration
.setDistortionCoefficientsMat(lensCalibration.getDistortionCoefficients());
calibration.setEnabled(true);
lensCalibration.close();
lensCalibration = null;
calibrating = false;
}
else {
calibrationCallback.callback(lensCalibration.getPatternFoundCount(),
calibrationCountGoal, false);
}
}
return appliedMat;
}
public void startCalibration(CalibrationCallback callback) {
this.calibrationCallback = callback;
calibration.setEnabled(false);
lensCalibration = new LensCalibration(LensModel.Pinhole, Pattern.AsymmetricCirclesGrid, 4,
11, 15, 750);
calibrating = true;
}
public void cancelCalibration() {
if (calibrating) {
lensCalibration.close();
}
calibrating = false;
}
public LensCalibrationParams getCalibration() {
return calibration;
}
@Override
public Location getLocation() {
// If this is a fixed camera we just treat the head offsets as it's
// table location.
if (getHead() == null) {
return getHeadOffsets();
}
return getDriver().getLocation(this);
}
public Length getSafeZ() {
return safeZ;
}
public void setSafeZ(Length safeZ) {
this.safeZ = safeZ;
}
@Override
public void close() throws IOException {}
@Override
public PropertySheet[] getPropertySheets() {
return new PropertySheet[] {
new PropertySheetWizardAdapter(new CameraConfigurationWizard(this), "General Configuration"),
new PropertySheetWizardAdapter(getConfigurationWizard(), "Camera Specific"),
new PropertySheetWizardAdapter(visionProvider.getConfigurationWizard(), "Vision Provider")};
}
@Override
public Action[] getPropertySheetHolderActions() {
return new Action[] { deleteAction };
}
public Action deleteAction = new AbstractAction("Delete Camera") {
{
putValue(SMALL_ICON, Icons.delete);
putValue(NAME, "Delete Camera");
putValue(SHORT_DESCRIPTION, "Delete the currently selected camera.");
}
@Override
public void actionPerformed(ActionEvent arg0) {
int ret = JOptionPane.showConfirmDialog(MainFrame.get(),
"Are you sure you want to delete " + getName() + "?",
"Delete " + getName() + "?", JOptionPane.YES_NO_OPTION);
if (ret == JOptionPane.YES_OPTION) {
if (getHead() != null) {
getHead().removeCamera(ReferenceCamera.this);
}
else {
Configuration.get().getMachine().removeCamera(ReferenceCamera.this);
}
MainFrame.get().getCameraViews().removeCamera(ReferenceCamera.this);
}
}
};
ReferenceDriver getDriver() {
return getMachine().getDriver();
}
ReferenceMachine getMachine() {
return (ReferenceMachine) Configuration.get().getMachine();
}
public interface CalibrationCallback {
public void callback(int progressCurrent, int progressMax, boolean complete);
}
public static class LensCalibrationParams {
@Attribute(required = false)
private boolean enabled = false;
@Element(name = "cameraMatrix", required = false)
private double[] cameraMatrixArr = new double[9];
@Element(name = "distortionCoefficients", required = false)
private double[] distortionCoefficientsArr = new double[5];
private Mat cameraMatrix = new Mat(3, 3, CvType.CV_64FC1);
private Mat distortionCoefficients = new Mat(5, 1, CvType.CV_64FC1);
@Commit
private void commit() {
cameraMatrix.put(0, 0, cameraMatrixArr);
distortionCoefficients.put(0, 0, distortionCoefficientsArr);
}
@Persist
private void persist() {
cameraMatrix.get(0, 0, cameraMatrixArr);
distortionCoefficients.get(0, 0, distortionCoefficientsArr);
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public Mat getCameraMatrixMat() {
return cameraMatrix;
}
public void setCameraMatrixMat(Mat cameraMatrix) {
this.cameraMatrix = cameraMatrix.clone();
}
public Mat getDistortionCoefficientsMat() {
return distortionCoefficients;
}
public void setDistortionCoefficientsMat(Mat distortionCoefficients) {
this.distortionCoefficients = distortionCoefficients.clone();
}
}
}