public/openhab-addons
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[pulseaudio] Add reencoding to play more audio formats (#11630) (#11631)

Browse Source 
Add a pass to reencode PCM sound in 16 bit, 44100 hz, 2 channels, before sending it to the pulseaudio audio sink.

Signed-off-by: Gwendal Roulleau <gwendal.roulleau@gmail.com>

Co-authored-by: Gwendal Roulleau <gwendal.roulleau@gmail.com>
This commit is contained in:
dalgwen 2021-12-11 13:08:03 +01:00 committed by GitHub
parent 15b83cc40f
commit 374a89a9fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 267 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

257
bundles/org.openhab.binding.pulseaudio/src/main/java/org/openhab/binding/pulseaudio/internal/ConvertedInputStream.java Normal file
View File

@ -0,0 +1,257 @@
/**
* Copyright (c) 2010-2021 Contributors to the openHAB project
*
* See the NOTICE file(s) distributed with this work for additional
* information.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.openhab.binding.pulseaudio.internal;
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
import javazoom.spi.mpeg.sampled.convert.MpegFormatConversionProvider;
import javazoom.spi.mpeg.sampled.file.MpegAudioFileReader;
import javax.sound.sampled.AudioFileFormat;
import javax.sound.sampled.AudioInputStream;
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.UnsupportedAudioFileException;
import org.eclipse.jdt.annotation.NonNullByDefault;
import org.eclipse.jdt.annotation.Nullable;
import org.openhab.core.audio.AudioFormat;
import org.openhab.core.audio.AudioStream;
import org.openhab.core.audio.FixedLengthAudioStream;
import org.openhab.core.audio.UnsupportedAudioFormatException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.tritonus.share.sampled.file.TAudioFileFormat;
/**
* This class convert a stream to the normalized pcm
* format wanted by the pulseaudio sink
*
* @author Gwendal Roulleau - Initial contribution
*/
@NonNullByDefault
public class ConvertedInputStream extends InputStream {
private final Logger logger = LoggerFactory.getLogger(ConvertedInputStream.class);
private static final javax.sound.sampled.AudioFormat TARGET_FORMAT = new javax.sound.sampled.AudioFormat(
javax.sound.sampled.AudioFormat.Encoding.PCM_SIGNED, 44100, 16, 2, 4, 44100, false);
private final AudioFormat audioFormat;
private AudioInputStream pcmNormalizedInputStream;
private long duration = -1;
private long length = -1;
public ConvertedInputStream(AudioStream innerInputStream)
throws UnsupportedAudioFormatException, UnsupportedAudioFileException, IOException {
this.audioFormat = innerInputStream.getFormat();
if (innerInputStream instanceof FixedLengthAudioStream) {
length = ((FixedLengthAudioStream) innerInputStream).length();
}
pcmNormalizedInputStream = getPCMStreamNormalized(getPCMStream(new ResetableInputStream(innerInputStream)));
}
@Override
public int read(byte @Nullable [] b) throws IOException {
return pcmNormalizedInputStream.read(b);
}
@Override
public int read(byte @Nullable [] b, int off, int len) throws IOException {
return pcmNormalizedInputStream.read(b, off, len);
}
@Override
public byte[] readAllBytes() throws IOException {
return pcmNormalizedInputStream.readAllBytes();
}
@Override
public byte[] readNBytes(int len) throws IOException {
return pcmNormalizedInputStream.readNBytes(len);
}
@Override
public int readNBytes(byte @Nullable [] b, int off, int len) throws IOException {
return pcmNormalizedInputStream.readNBytes(b, off, len);
}
@Override
public int read() throws IOException {
return pcmNormalizedInputStream.read();
}
@Override
public void close() throws IOException {
pcmNormalizedInputStream.close();
}
/**
* Ensure right PCM format by converting if needed (sample rate, channel)
*
* @param pcmInputStream
*
* @return A PCM normalized stream (2 channel, 44100hz, 16 bit signed)
*/
private AudioInputStream getPCMStreamNormalized(AudioInputStream pcmInputStream) {
javax.sound.sampled.AudioFormat format = pcmInputStream.getFormat();
if (format.getChannels() != 2
|| !format.getEncoding().equals(javax.sound.sampled.AudioFormat.Encoding.PCM_SIGNED)
|| Math.abs(format.getFrameRate() - 44100) > 1000) {
logger.debug("Sound is not in the target format. Trying to reencode it");
return AudioSystem.getAudioInputStream(TARGET_FORMAT, pcmInputStream);
} else {
return pcmInputStream;
}
}
public long getDuration() {
return duration;
}
/**
* If necessary, this method convert MP3 to PCM, and try to
* extract duration information.
*
* @param resetableInnerInputStream A stream supporting reset operation
* (reset is mandatory to parse formation without loosing data)
*
* @return PCM stream
* @throws UnsupportedAudioFileException
* @throws IOException
* @throws UnsupportedAudioFormatException
*/
private AudioInputStream getPCMStream(InputStream resetableInnerInputStream)
throws UnsupportedAudioFileException, IOException, UnsupportedAudioFormatException {
if (AudioFormat.MP3.isCompatible(audioFormat)) {
MpegAudioFileReader mpegAudioFileReader = new MpegAudioFileReader();
if (length > 0) { // compute duration if possible
AudioFileFormat audioFileFormat = mpegAudioFileReader.getAudioFileFormat(resetableInnerInputStream);
if (audioFileFormat instanceof TAudioFileFormat) {
Map<String, Object> taudioFileFormatProperties = ((TAudioFileFormat) audioFileFormat).properties();
if (taudioFileFormatProperties.containsKey("mp3.framesize.bytes")
&& taudioFileFormatProperties.containsKey("mp3.framerate.fps")) {
Integer frameSize = (Integer) taudioFileFormatProperties.get("mp3.framesize.bytes");
Float frameRate = (Float) taudioFileFormatProperties.get("mp3.framerate.fps");
if (frameSize != null && frameRate != null) {
duration = Math.round((length / (frameSize * frameRate)) * 1000);
logger.debug("Duration of input stream : {}", duration);
}
}
}
resetableInnerInputStream.reset();
}
logger.debug("Sound is a MP3. Trying to reencode it");
// convert MP3 to PCM :
AudioInputStream sourceAIS = mpegAudioFileReader.getAudioInputStream(resetableInnerInputStream);
javax.sound.sampled.AudioFormat sourceFormat = sourceAIS.getFormat();
MpegFormatConversionProvider mpegconverter = new MpegFormatConversionProvider();
javax.sound.sampled.AudioFormat convertFormat = new javax.sound.sampled.AudioFormat(
javax.sound.sampled.AudioFormat.Encoding.PCM_SIGNED, sourceFormat.getSampleRate(), 16,
sourceFormat.getChannels(), sourceFormat.getChannels() * 2, sourceFormat.getSampleRate(), false);
return mpegconverter.getAudioInputStream(convertFormat, sourceAIS);
} else if (AudioFormat.WAV.isCompatible(audioFormat)) {
// return the same input stream, but try to compute the duration first
AudioInputStream audioInputStream = AudioSystem.getAudioInputStream(resetableInnerInputStream);
if (length > 0) {
int frameSize = audioInputStream.getFormat().getFrameSize();
float frameRate = audioInputStream.getFormat().getFrameRate();
float durationInSeconds = (length / (frameSize * frameRate));
duration = Math.round(durationInSeconds * 1000);
logger.debug("Duration of input stream : {}", duration);
}
return audioInputStream;
} else {
throw new UnsupportedAudioFormatException("Pulseaudio audio sink can only play pcm or mp3 stream",
audioFormat);
}
}
/**
* This class add reset capability (on the first bytes only)
* to an AudioStream. This is necessary for the parsing / format detection.
*
*/
public static class ResetableInputStream extends InputStream {
private static final int BUFFER_LENGTH = 10000;
private final InputStream originalInputStream;
private int position = -1;
private int markPosition = -1;
private int maxPreviousPosition = -2;
private byte[] startingBuffer = new byte[BUFFER_LENGTH + 1];
public ResetableInputStream(InputStream originalInputStream) {
this.originalInputStream = originalInputStream;
}
@Override
public void close() throws IOException {
originalInputStream.close();
}
@Override
public int read() throws IOException {
if (position >= BUFFER_LENGTH || originalInputStream.markSupported()) {
return originalInputStream.read();
} else {
position++;
if (position <= maxPreviousPosition) {
return Byte.toUnsignedInt(startingBuffer[position]);
} else {
int currentByte = originalInputStream.read();
startingBuffer[position] = (byte) currentByte;
maxPreviousPosition = position;
return currentByte;
}
}
}
@Override
public synchronized void mark(int readlimit) {
if (originalInputStream.markSupported()) {
originalInputStream.mark(readlimit);
}
markPosition = position;
}
@Override
public boolean markSupported() {
return true;
}
@Override
public synchronized void reset() throws IOException {
if (originalInputStream.markSupported()) {
originalInputStream.reset();
} else if (position >= BUFFER_LENGTH) {
throw new IOException("mark/reset not supported above " + BUFFER_LENGTH + " bytes");
}
position = markPosition;
}
}
}

118
bundles/org.openhab.binding.pulseaudio/src/main/java/org/openhab/binding/pulseaudio/internal/PulseAudioAudioSink.java
View File

@ -13,23 +13,16 @@
package org.openhab.binding.pulseaudio.internal;
import java.io.IOException;
import java.io.InputStream;
import java.net.Socket;
import java.time.Duration;
import java.time.Instant;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import javazoom.spi.mpeg.sampled.convert.MpegFormatConversionProvider;
import javazoom.spi.mpeg.sampled.file.MpegAudioFileReader;
import javax.sound.sampled.AudioFileFormat;
import javax.sound.sampled.AudioInputStream;
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.UnsupportedAudioFileException;
import org.eclipse.jdt.annotation.NonNullByDefault;
@ -44,7 +37,6 @@ import org.openhab.core.audio.UnsupportedAudioStreamException;
import org.openhab.core.library.types.PercentType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.tritonus.share.sampled.file.TAudioFileFormat;
/**
* The audio sink for openhab, implemented by a connection to a pulseaudio sink
@ -90,52 +82,6 @@ public class PulseAudioAudioSink implements AudioSink {
return pulseaudioHandler.getThing().getLabel();
}
/**
* Convert MP3 to PCM, as this is the only possible format
*
* @param input
* @return
*/
private @Nullable AudioStreamAndDuration getPCMStreamFromMp3Stream(InputStream input) {
try {
MpegAudioFileReader mpegAudioFileReader = new MpegAudioFileReader();
int duration = -1;
if (input instanceof FixedLengthAudioStream) {
final Long audioFileLength = ((FixedLengthAudioStream) input).length();
AudioFileFormat audioFileFormat = mpegAudioFileReader.getAudioFileFormat(input);
if (audioFileFormat instanceof TAudioFileFormat) {
Map<String, Object> taudioFileFormatProperties = ((TAudioFileFormat) audioFileFormat).properties();
if (taudioFileFormatProperties.containsKey("mp3.framesize.bytes")
&& taudioFileFormatProperties.containsKey("mp3.framerate.fps")) {
Integer frameSize = (Integer) taudioFileFormatProperties.get("mp3.framesize.bytes");
Float frameRate = (Float) taudioFileFormatProperties.get("mp3.framerate.fps");
if (frameSize != null && frameRate != null) {
duration = Math.round((audioFileLength / (frameSize * frameRate)) * 1000);
}
}
}
input.reset();
}
AudioInputStream sourceAIS = mpegAudioFileReader.getAudioInputStream(input);
javax.sound.sampled.AudioFormat sourceFormat = sourceAIS.getFormat();
MpegFormatConversionProvider mpegconverter = new MpegFormatConversionProvider();
javax.sound.sampled.AudioFormat convertFormat = new javax.sound.sampled.AudioFormat(
javax.sound.sampled.AudioFormat.Encoding.PCM_SIGNED, sourceFormat.getSampleRate(), 16,
sourceFormat.getChannels(), sourceFormat.getChannels() * 2, sourceFormat.getSampleRate(), false);
AudioInputStream audioInputStreamConverted = mpegconverter.getAudioInputStream(convertFormat, sourceAIS);
return new AudioStreamAndDuration(audioInputStreamConverted, duration);
} catch (IOException | UnsupportedAudioFileException e) {
logger.warn("Cannot convert this mp3 stream to pcm stream: {}", e.getMessage());
}
return null;
}
/**
* Connect to pulseaudio with the simple protocol
*
@ -168,23 +114,6 @@ public class PulseAudioAudioSink implements AudioSink {
}
}
private AudioStreamAndDuration getWavAudioAndDuration(AudioStream audioStream) {
int duration = -1;
if (audioStream instanceof FixedLengthAudioStream) {
final Long audioFileLength = ((FixedLengthAudioStream) audioStream).length();
try {
AudioInputStream audioInputStream = AudioSystem.getAudioInputStream(audioStream);
int frameSize = audioInputStream.getFormat().getFrameSize();
float frameRate = audioInputStream.getFormat().getFrameRate();
float durationInSeconds = (audioFileLength / (frameSize * frameRate));
duration = Math.round(durationInSeconds * 1000);
} catch (UnsupportedAudioFileException | IOException e) {
logger.warn("Error when getting duration information from AudioFile");
}
}
return new AudioStreamAndDuration(audioStream, duration);
}
@Override
public void process(@Nullable AudioStream audioStream)
throws UnsupportedAudioFormatException, UnsupportedAudioStreamException {
@ -193,34 +122,22 @@ public class PulseAudioAudioSink implements AudioSink {
return;
}
AudioStreamAndDuration audioInputStreamAndDuration = null;
try {
if (AudioFormat.MP3.isCompatible(audioStream.getFormat())) {
audioInputStreamAndDuration = getPCMStreamFromMp3Stream(audioStream);
} else if (AudioFormat.WAV.isCompatible(audioStream.getFormat())) {
audioInputStreamAndDuration = getWavAudioAndDuration(audioStream);
} else {
throw new UnsupportedAudioFormatException("pulseaudio audio sink can only play pcm or mp3 stream",
audioStream.getFormat());
}
try (ConvertedInputStream normalizedPCMStream = new ConvertedInputStream(audioStream)) {
for (int countAttempt = 1; countAttempt <= 2; countAttempt++) { // two attempts allowed
try {
connectIfNeeded();
final Socket clientSocketLocal = clientSocket;
if (audioInputStreamAndDuration != null && clientSocketLocal != null) {
if (clientSocketLocal != null) {
// send raw audio to the socket and to pulse audio
isIdle = false;
Instant start = Instant.now();
audioInputStreamAndDuration.inputStream.transferTo(clientSocketLocal.getOutputStream());
if (audioInputStreamAndDuration.duration != -1) { // ensure, if the sound has a duration
normalizedPCMStream.transferTo(clientSocketLocal.getOutputStream());
if (normalizedPCMStream.getDuration() != -1) { // ensure, if the sound has a duration
// that we let at least this time for the system to play
Instant end = Instant.now();
long millisSecondTimedToSendAudioData = Duration.between(start, end).toMillis();
if (millisSecondTimedToSendAudioData < audioInputStreamAndDuration.duration) {
long timeToSleep = audioInputStreamAndDuration.duration
- millisSecondTimedToSendAudioData;
if (millisSecondTimedToSendAudioData < normalizedPCMStream.getDuration()) {
long timeToSleep = normalizedPCMStream.getDuration() - millisSecondTimedToSendAudioData;
logger.debug("Sleep time to let the system play sound : {}", timeToSleep);
Thread.sleep(timeToSleep);
}
@ -243,15 +160,11 @@ public class PulseAudioAudioSink implements AudioSink {
break;
}
}
} catch (UnsupportedAudioFileException | IOException e) {
throw new UnsupportedAudioFormatException("Cannot send sound to the pulseaudio sink",
audioStream.getFormat(), e);
} finally {
try {
if (audioInputStreamAndDuration != null) {
audioInputStreamAndDuration.inputStream.close();
}
audioStream.close();
scheduleDisconnect();
} catch (IOException e) {
}
scheduleDisconnect();
}
isIdle = true;
}
@ -286,15 +199,4 @@ public class PulseAudioAudioSink implements AudioSink {
public void setVolume(PercentType volume) {
pulseaudioHandler.setVolume(volume.intValue());
}
private static class AudioStreamAndDuration {
private InputStream inputStream;
private int duration;
public AudioStreamAndDuration(InputStream inputStream, int duration) {
super();
this.inputStream = inputStream;
this.duration = duration + 200; // introduce some delay
}
}
}