/**
* Copyright 2010 DFKI GmbH.
* All Rights Reserved. Use is subject to license terms.
*
* This file is part of MARY TTS.
*
* MARY TTS is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, version 3 of the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
package marytts.unitselection.analysis;
import java.util.Arrays;
import marytts.modules.phonemiser.Allophone;
import marytts.unitselection.concat.BaseUnitConcatenator.UnitData;
import marytts.unitselection.select.HalfPhoneTarget;
import marytts.unitselection.select.SelectedUnit;
import marytts.util.data.Datagram;
import marytts.util.math.MathUtils;
import org.apache.commons.lang.ArrayUtils;
import org.w3c.dom.Element;
/**
* Convenience class containing the selected units and targets of a phone segment, and a host of getters to access their prosodic
* attributes
*
* @author steiner
*
*/
public class Phone {
private HalfPhoneTarget leftTarget;
private HalfPhoneTarget rightTarget;
private SelectedUnit leftUnit;
private SelectedUnit rightUnit;
private double sampleRate;
private double[] leftF0Targets;
private double[] rightF0Targets;
/**
* Main constructor
*
* @param leftUnit
* which can be null
* @param rightUnit
* which can be null
* @param sampleRate
* of the TimelineReader containing the SelectedUnits, needed to provide duration information
* @throws IllegalArgumentException
* if both the left and right units are null
*/
public Phone(SelectedUnit leftUnit, SelectedUnit rightUnit, int sampleRate) throws IllegalArgumentException {
this.leftUnit = leftUnit;
this.rightUnit = rightUnit;
this.sampleRate = (double) sampleRate;
// targets are extracted from the units for easier access:
try {
this.leftTarget = (HalfPhoneTarget) leftUnit.getTarget();
} catch (NullPointerException e) {
// leave at null
}
try {
this.rightTarget = (HalfPhoneTarget) rightUnit.getTarget();
} catch (NullPointerException e) {
if (leftTarget == null) {
throw new IllegalArgumentException("A phone's left and right halves cannot both be null!");
} else {
// leave at null
}
}
}
/**
* Get the left selected halfphone unit of this phone
*
* @return the left unit, or null if there is no left unit
*/
public SelectedUnit getLeftUnit() {
return leftUnit;
}
/**
* Get the right selected halfphone unit of this phone
*
* @return the right unit, or null if there is no right unit
*/
public SelectedUnit getRightUnit() {
return rightUnit;
}
/**
* Get the left halfphone target of this phone
*
* @return the left target, or null if there is no left target
*/
public HalfPhoneTarget getLeftTarget() {
return leftTarget;
}
/**
* Get the right halfphone target of this phone
*
* @return the right target, or null if there is no right target
*/
public HalfPhoneTarget getRightTarget() {
return rightTarget;
}
/**
* Get the datagrams from a SelectedUnit
*
* @param unit
* the SelectedUnit
* @return the datagrams in an array, or null if unit is null
*/
private Datagram[] getUnitFrames(SelectedUnit unit) {
UnitData unitData = getUnitData(unit);
Datagram[] frames = null;
try {
frames = unitData.getFrames();
} catch (NullPointerException e) {
// leave at null
}
return frames;
}
/**
* Get this phone's left unit's Datagrams
*
* @return the left unit's Datagrams in an array, or null if there is no left unit
*/
public Datagram[] getLeftUnitFrames() {
return getUnitFrames(leftUnit);
}
/**
* Get this phone's right unit's Datagrams
*
* @return the right unit's Datagrams in an array, or null if there is no right unit
*/
public Datagram[] getRightUnitFrames() {
return getUnitFrames(rightUnit);
}
/**
* Get all Datagrams in this phone's units
*
* @return the left and right unit's Datagrams in an array
*/
public Datagram[] getUnitDataFrames() {
Datagram[] leftUnitFrames = getLeftUnitFrames();
Datagram[] rightUnitFrames = getRightUnitFrames();
Datagram[] frames = (Datagram[]) ArrayUtils.addAll(leftUnitFrames, rightUnitFrames);
return frames;
}
/**
* Get the number of Datagrams in a SelectedUnit's UnitData
*
* @param unit
* whose Datagrams to count
* @return the number of Datagrams in the unit, or 0 if unit is null
*/
private int getNumberOfUnitFrames(SelectedUnit unit) {
int numberOfFrames = 0;
try {
Datagram[] frames = getUnitData(unit).getFrames();
numberOfFrames = frames.length;
} catch (NullPointerException e) {
// leave at 0
}
return numberOfFrames;
}
/**
* Get the number of Datagrams in this phone's left unit
*
* @return the number of Datagrams in the left unit, or 0 if there is no left unit
*/
public int getNumberOfLeftUnitFrames() {
return getNumberOfUnitFrames(leftUnit);
}
/**
* Get the number of Datagrams in this phone's right unit
*
* @return the number of Datagrams in the right unit, or 0 if there is no right unit
*/
public int getNumberOfRightUnitFrames() {
return getNumberOfUnitFrames(rightUnit);
}
/**
* Get the number of Datagrams in this phone's left and right units
*
* @return the number of Datagrams in this phone
*/
public int getNumberOfFrames() {
return getNumberOfLeftUnitFrames() + getNumberOfRightUnitFrames();
}
/**
* Get the durations (in seconds) of each Datagram in this phone's units
*
* @return the left and right unit's Datagram durations (in seconds) in an array
*/
public double[] getFrameDurations() {
Datagram[] frames = getUnitDataFrames();
double[] durations = new double[frames.length];
for (int f = 0; f < frames.length; f++) {
durations[f] = frames[f].getDuration() / sampleRate;
}
return durations;
}
/**
* Get the target duration (in seconds) of a HalfPhoneTarget
*
* @param target
* the target whose duration to get
* @return the target duration in seconds, or 0 if target is null
*/
private double getTargetDuration(HalfPhoneTarget target) {
double duration = 0;
try {
duration = target.getTargetDurationInSeconds();
} catch (NullPointerException e) {
// leave at 0
}
return duration;
}
/**
* Get this phone's left target's duration (in seconds)
*
* @return the left target's duration in seconds, or 0 if there is no left target
*/
public double getLeftTargetDuration() {
return getTargetDuration(leftTarget);
}
/**
* Get this phone's right target's duration (in seconds)
*
* @return the right target's duration in seconds, or 0 if there is no right target
*/
public double getRightTargetDuration() {
return getTargetDuration(rightTarget);
}
/**
* Get the predicted duration of this phone (which is the sum of the left and right target's duration)
*
* @return the predicted phone duration in seconds
*/
public double getPredictedDuration() {
return getLeftTargetDuration() + getRightTargetDuration();
}
/**
* Convenience getter for a SelectedUnit's UnitData
*
* @param unit
* the unit whose UnitData to get
* @return the UnitData of the unit, or null, if unit is null
*/
private UnitData getUnitData(SelectedUnit unit) {
UnitData unitData = null;
try {
unitData = (UnitData) unit.getConcatenationData();
} catch (NullPointerException e) {
// leave at null
}
return unitData;
}
/**
* Get this phone's left unit's UnitData
*
* @return the left unit's UnitData, or null if there is no left unit
*/
public UnitData getLeftUnitData() {
return getUnitData(leftUnit);
}
/**
* Get this phone's right unit's UnitData
*
* @return the right unit's UnitData, or null if there is no right unit
*/
public UnitData getRightUnitData() {
return getUnitData(rightUnit);
}
/**
* Get the actual duration (in seconds) of a SelectedUnit, from its UnitData
*
* @param unit
* whose duration to get
* @return the unit's duration in seconds, or 0 if unit is null
*/
private double getUnitDuration(SelectedUnit unit) {
int durationInSamples = 0;
try {
durationInSamples = getUnitData(unit).getUnitDuration();
} catch (NullPointerException e) {
// leave at 0
}
double duration = durationInSamples / sampleRate;
return duration;
}
/**
* Get this phone's left unit's duration (in seconds)
*
* @return the left unit's duration in seconds, or 0 if there is no left unit
*/
public double getLeftUnitDuration() {
return getUnitDuration(leftUnit);
}
/**
* Get this phone's right unit's duration (in seconds)
*
* @return the right unit's duration in seconds, or 0 if there is no right unit
*/
public double getRightUnitDuration() {
return getUnitDuration(rightUnit);
}
/**
* Get the realized duration (in seconds) of this phone (which is the sum of the durations of the left and right units)
*
* @return the phone's realized duration in seconds
*/
public double getRealizedDuration() {
return getLeftUnitDuration() + getRightUnitDuration();
}
/**
* Get the factor needed to convert the realized duration of a unit to the target duration
*
* @param unit
* whose realized duration to convert
* @param target
* whose duration to match
* @return the multiplication factor to convert from the realized duration to the target duration, or 0 if the unit duration
* is 0
*/
private double getDurationFactor(SelectedUnit unit, HalfPhoneTarget target) {
double unitDuration = getUnitDuration(unit);
if (unitDuration <= 0) {
// throw new ArithmeticException("Realized duration must be greater than 0!");
return 0;
}
double targetDuration = getTargetDuration(target);
double durationFactor = targetDuration / unitDuration;
return durationFactor;
}
/**
* Get the factor to convert this phone's left unit's duration into this phone's left target duration
*
* @return the left duration factor
*/
public double getLeftDurationFactor() {
return getDurationFactor(leftUnit, leftTarget);
}
/**
* Get the factor to convert this phone's right unit's duration into this phone's right target duration
*
* @return the right duration factor
*/
public double getRightDurationFactor() {
return getDurationFactor(rightUnit, rightTarget);
}
/**
* Get the duration factors for this phone, one per datagram. Each factor corresponding to a datagram in the left unit is that
* required to convert the left unit's duration to the left target duration, and likewise for the right unit's datagrams.
*
* @return the left and right duration factors for this phone, in an array whose size matched the Datagrams in this phone's
* units
*/
public double[] getFramewiseDurationFactors() {
double[] durationFactors = new double[getNumberOfFrames()];
int numberOfLeftUnitFrames = getNumberOfLeftUnitFrames();
double leftDurationFactor = getLeftDurationFactor();
Arrays.fill(durationFactors, 0, numberOfLeftUnitFrames, leftDurationFactor);
double rightDurationFactor = getRightDurationFactor();
Arrays.fill(durationFactors, numberOfLeftUnitFrames, getNumberOfFrames(), rightDurationFactor);
return durationFactors;
}
/**
* Set the target F0 values of this phone's left half, with one value per Datagram in the phone's left unit
*
* @param f0TargetValues
* array of target F0 values to assign to the left halfphone
* @throws IllegalArgumentException
* if the length of f0TargetValues does not match the number of Datagrams in the phone's left unit
*/
public void setLeftTargetF0Values(double[] f0TargetValues) throws IllegalArgumentException {
int numberOfLeftUnitFrames = getNumberOfLeftUnitFrames();
if (f0TargetValues.length != numberOfLeftUnitFrames) {
throw new IllegalArgumentException("Wrong number of F0 targets (" + f0TargetValues.length
+ ") for number of frames (" + numberOfLeftUnitFrames + " in halfphone: '" + leftUnit.toString() + "'");
}
this.leftF0Targets = f0TargetValues;
}
/**
* Set the target F0 values of this phone's right half, with one value per Datagram in the phone's right unit
*
* @param f0TargetValues
* array of target F0 values to assign to the right halfphone
* @throws IllegalArgumentException
* if the length of f0TargetValues does not match the number of Datagrams in the phone's right unit
*/
public void setRightTargetF0Values(double[] f0TargetValues) {
if (f0TargetValues.length != getNumberOfRightUnitFrames()) {
throw new IllegalArgumentException("Wrong number of F0 targets (" + f0TargetValues.length
+ ") for number of frames (" + getNumberOfRightUnitFrames() + " in halfphone: '" + rightUnit.toString() + "'");
}
this.rightF0Targets = f0TargetValues;
}
/**
* Get the target F0 values for this phone's left half, with one value per Datagram in the phone's left unit
*
* @return the target F0 values for the left halfphone in an array
* @throws NullPointerException
* if the target F0 values for the left halfphone are null
*/
public double[] getLeftTargetF0Values() throws NullPointerException {
if (leftF0Targets == null) {
throw new NullPointerException("The left target F0 values have not been assigned!");
}
return leftF0Targets;
}
/**
* Get the target F0 values for this phone's right half, with one value per Datagram in the phone's right unit
*
* @return the target F0 values for the right halfphone in an array
* @throws NullPointerException
* if the target F0 values for the right halfphone are null
*/
public double[] getRightTargetF0Values() throws NullPointerException {
if (rightF0Targets == null) {
throw new NullPointerException("The right target F0 values have not been assigned!");
}
return rightF0Targets;
}
/**
* Get the target F0 values for this phone, with one value per Datagram in the phone's left and right units
*
* @return the target F0 values with one value per Datagram in an array
*/
public double[] getTargetF0Values() {
double[] f0Targets = ArrayUtils.addAll(leftF0Targets, rightF0Targets);
return f0Targets;
}
/**
* Get the mean target F0 for this phone
*
* @return the mean predicted F0 value
*/
public double getPredictedF0() {
double meanF0 = MathUtils.mean(getTargetF0Values());
return meanF0;
}
/**
* Get the durations (in seconds) of each Datagram in a SelectedUnit's UnitData
*
* @param unit
* whose Datagrams' durations to get
* @return the durations (in seconds) of each Datagram in the unit in an array, or null if unit is null
*/
private double[] getUnitFrameDurations(SelectedUnit unit) {
Datagram[] frames = null;
try {
frames = getUnitData(unit).getFrames();
} catch (NullPointerException e) {
return null;
}
assert frames != null;
double[] frameDurations = new double[frames.length];
for (int f = 0; f < frames.length; f++) {
long frameDuration = frames[f].getDuration();
frameDurations[f] = frameDuration / sampleRate; // converting to seconds
}
return frameDurations;
}
/**
* Get the durations (in seconds) of each Datagram in this phone's left unit
*
* @return the durations of each Datagram in the left unit in an array, or null if there is no left unit
*/
public double[] getLeftUnitFrameDurations() {
return getUnitFrameDurations(leftUnit);
}
/**
* Get the durations (in seconds) of each Datagram in this phone's right unit
*
* @return the durations of each Datagram in the right unit in an array, or null if there is no right unit
*/
public double[] getRightUnitFrameDurations() {
return getUnitFrameDurations(rightUnit);
}
/**
* Get the durations (in seconds) of each Datagram in this phone's left and right units
*
* @return the durations of all Datagrams in this phone in an array
*/
public double[] getRealizedFrameDurations() {
return ArrayUtils.addAll(getLeftUnitFrameDurations(), getRightUnitFrameDurations());
}
/**
* Get the F0 values from a SelectedUnit's Datagrams.
* <p>
* Since these are not stored explicitly, we are forced to rely on the inverse of the Datagram durations, which means that
* during unvoiced regions, we recover a value of 100 Hz...
*
* @param unit
* from whose Datagrams to recover the F0 values
* @return the F0 value of each Datagram in the unit in an array, or null if the unit is null or any of the Datagrams have
* zero duration
*/
private double[] getUnitF0Values(SelectedUnit unit) {
double[] f0Values = null;
try {
double[] durations = getUnitFrameDurations(unit);
if (!ArrayUtils.contains(durations, 0)) {
f0Values = MathUtils.invert(durations);
} else {
// leave at null
}
} catch (IllegalArgumentException e) {
// leave at null
}
return f0Values;
}
/**
* Recover the F0 values from this phone's left unit's Datagrams
*
* @return the left unit's F0 values in an array, with one value per Datagram, or null if there is no left unit or any of the
* left unit's Datagrams have zero duration
*/
public double[] getLeftUnitFrameF0s() {
return getUnitF0Values(leftUnit);
}
/**
* Recover the F0 values from this phone's right unit's Datagrams
*
* @return the right unit's F0 values in an array, with one value per Datagram, or null if there is no right unit or any of
* the right unit's Datagrams have zero duration
*/
public double[] getRightUnitFrameF0s() {
return getUnitF0Values(rightUnit);
}
/**
* Recover the F0 values from each Datagram in this phone's left and right units
*
* @return the F0 values for each Datagram in this phone, or null if either the left or the right unit contain a Datagram with
* zero duration
*/
public double[] getUnitFrameF0s() {
double[] leftUnitFrameF0s = getLeftUnitFrameF0s();
double[] rightUnitFrameF0s = getRightUnitFrameF0s();
double[] unitFrameF0s = ArrayUtils.addAll(leftUnitFrameF0s, rightUnitFrameF0s);
return unitFrameF0s;
}
/**
* Get the realized F0 by recovering the F0 from all Datagrams in this phone and computing the mean
*
* @return the mean F0 of all Datagrams in this phone's left and right units
*/
public double getRealizedF0() {
double meanF0 = MathUtils.mean(getUnitFrameF0s());
return meanF0;
}
/**
* Get the factors required to convert the F0 values recovered from the Datagrams in a SelectedUnit to the target F0 values.
*
* @param unit
* from which to recover the realized F0 values
* @param target
* for which to get the target F0 values
* @return each Datagram's F0 factor in an array, or null if realized and target F0 values differ in length
* @throws ArithmeticException
* if any of the F0 values recovered from the unit's Datagrams is zero
*/
private double[] getUnitF0Factors(SelectedUnit unit, HalfPhoneTarget target) throws ArithmeticException {
double[] unitF0Values = getUnitF0Values(unit);
if (ArrayUtils.contains(unitF0Values, 0)) {
throw new ArithmeticException("Unit frames must not have F0 of 0!");
}
double[] targetF0Values;
if (target == null || target.isLeftHalf()) {
targetF0Values = getLeftTargetF0Values();
} else {
targetF0Values = getRightTargetF0Values();
}
double[] f0Factors = null;
try {
f0Factors = MathUtils.divide(targetF0Values, unitF0Values);
} catch (IllegalArgumentException e) {
// leave at null
}
return f0Factors;
}
/**
* Get the factors to convert each of the F0 values in this phone's left half to the corresponding target value
*
* @return the F0 factors for this phone's left half in an array with one value per Datagram, or null if the number of
* Datagrams does not match the number of left target F0 values
*/
public double[] getLeftF0Factors() {
return getUnitF0Factors(leftUnit, leftTarget);
}
/**
* Get the factors to convert each of the F0 values in this phone's right half to the corresponding target value
*
* @return the F0 factors for this phone's right half in an array with one value per Datagram, or null if the number of
* Datagrams does not match the number of right target F0 values
*/
public double[] getRightF0Factors() {
return getUnitF0Factors(rightUnit, rightTarget);
}
/**
* Get the F0 factor for each Datagram in this phone's left and right units
*
* @return the F0 factors, in an array with one value per Datagram
*/
public double[] getF0Factors() {
return ArrayUtils.addAll(getLeftF0Factors(), getRightF0Factors());
}
/**
* Get the Allophone represented by this
*
* @return the Allophone
*/
private Allophone getAllophone() {
if (leftTarget != null) {
return leftTarget.getAllophone();
} else if (rightTarget != null) {
return rightTarget.getAllophone();
}
return null;
}
/**
* Determine whether this is a transient phone (i.e. a plosive)
*
* @return true if this is a plosive, false otherwise
*/
public boolean isTransient() {
Allophone allophone = getAllophone();
if (allophone.isPlosive() || allophone.isAffricate()) {
return true;
} else {
return false;
}
}
/**
* Determine whether this is a voiced phone
*
* @return true if this is voiced, false otherwise
*/
public boolean isVoiced() {
Allophone allophone = getAllophone();
if (allophone.isVoiced()) {
return true;
} else {
return false;
}
}
/**
* get the MaryXML Element corresponding to this Phone's Target
* <p>
* If both <b>leftTarget</b> and <b>rightTarget</b> are not <b>null</b>, their respective MaryXML Elements should be
* <i>equal</i>.
*
* @return the MaryXML Element, or null if both halfphone targets are null
*/
public Element getMaryXMLElement() {
if (leftTarget != null) {
return leftTarget.getMaryxmlElement();
} else if (rightTarget != null) {
return rightTarget.getMaryxmlElement();
}
return null;
}
/**
* for debugging, provide the names of the left and right targets as the string representation of this class
*/
public String toString() {
String string = "";
if (leftTarget != null) {
string += " " + leftTarget.getName();
}
if (rightTarget != null) {
string += " " + rightTarget.getName();
}
return string;
}
}