Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Web radio chunked demo #48

Open
wants to merge 13 commits into
base: master
Choose a base branch
from
225 changes: 225 additions & 0 deletions examples/WebRadioChunkedDemo/WebRadioChunkedDemo.ino
Original file line number Diff line number Diff line change
@@ -0,0 +1,225 @@
/**
* A chunked stream handler to play web radio stations using ESP8266.
* It shows how to remove chunk control data from the stream to avoid the sound glitches.
*
* More technical details about chunked transfer:
* https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Transfer-Encoding
* https://en.wikipedia.org/wiki/Chunked_transfer_encoding
*
* Github discussion related to this demo:
* https://github.com/baldram/ESP_VS1053_Library/issues/47
*/

#include <VS1053.h>
#ifdef ARDUINO_ARCH_ESP8266
#include <ESP8266WiFi.h>
#define VS1053_CS D1
#define VS1053_DCS D0
#define VS1053_DREQ D3
#endif

#ifdef ARDUINO_ARCH_ESP32
#include <WiFi.h>
#define VS1053_CS 5
#define VS1053_DCS 16
#define VS1053_DREQ 4
#endif

// Default volume
#define VOLUME 80

VS1053 player(VS1053_CS, VS1053_DCS, VS1053_DREQ);
WiFiClient client;

const char *ssid = "TP-Link";
const char *password = "xxxxxxxx";

// this is the audio stream which uses chunked transfer
const char *host = "icecast.radiofrance.fr";
const char *path = "/franceculture-lofi.mp3";
int httpPort = 80;

// The buffer size 64 seems to be optimal. At 32 and 128 the sound might be brassy.
#define BUFFER_SIZE 64
uint8_t mp3buff[BUFFER_SIZE];

uint8_t remove_chunk_control_data(uint8_t *data, size_t length);

void setup() {
Serial.begin(115200);

// Wait for VS1053 and PAM8403 to power up
// otherwise the system might not start up correctly
delay(3000);

// This can be set in the IDE no need for ext library
// system_update_cpu_freq(160);

Serial.println("\n\Chunked Transfer Radio Node WiFi Radio");

SPI.begin();

player.begin();
player.switchToMp3Mode();
player.setVolume(VOLUME);

Serial.print("Connecting to SSID ");
Serial.println(ssid);
WiFi.begin(ssid, password);
WiFi.setAutoConnect(true);
WiFi.setAutoReconnect(true);
WiFi.reconnect();

while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
}

Serial.println("WiFi connected");
Serial.println("IP address: ");
Serial.println(WiFi.localIP());

Serial.print("connecting to ");
Serial.println(host);

if (!client.connect(host, httpPort)) {
Serial.println("Connection failed");
return;
}

Serial.print("Requesting stream: ");
Serial.println(path);

client.print(String("GET ") + path + " HTTP/1.1\r\n" + "Host: " + host + "\r\n" + "Connection: close\r\n\r\n");
}

void loop() {
if (!client.connected()) {
Serial.println("Reconnecting...");
if (client.connect(host, httpPort)) {
client.print(String("GET ") + path + " HTTP/1.1\r\n" + "Host: " + host + "\r\n" + "Connection: close\r\n\r\n");
}
}

if (client.available() > 0) {
uint8_t bytesread = client.read(mp3buff, BUFFER_SIZE);
bytesread = remove_chunk_control_data(mp3buff, bytesread);
player.playChunk(mp3buff, bytesread);
}
}

// Introduce here a new helper buffer with size of 8 bytes, to remove the chunked control bytes.
// Variables must be aligned to avoid the Exception (9) being thrown from the ESP8266
// See here: https://arduino-esp8266.readthedocs.io/en/latest/exception_causes.html
// See here: https://arduino.stackexchange.com/questions/67442/nodemcu-1-0-exception-9-fatal-exception-9loadstorealignmentcause
uint8_t __attribute__((aligned(4))) helperBuffer[8];
uint8_t __attribute__((aligned(4))) helperBufferCount = 0;

/***
* Removes the chunk control data from the helper buffer.
*
* Only the following chunk control bytes are removed:
* \r\n<byte><byte>\r\n
* \r\n<byte><byte><byte>\r\n
* \r\n<byte><byte><byte><byte>\r\n
*
*/
void remove_chunk_control_data_from_helper_buffer() {
if (helperBuffer[0] != '\r') {
// die fast
return;
}
if (helperBuffer[1] != '\n') {
// die fast
return;
}

if (helperBuffer[4] == '\r' && helperBuffer[5] == '\n') {
// 6 bytes length chunk control section discovered
// \r\n<byte><byte>\r\n
helperBufferCount = 2;
Serial.println("Removed control data: 6 bytes");

return;
}

if (helperBuffer[5] == '\r' && helperBuffer[6] == '\n') {
// 7 bytes length chunk control section discovered
// \r\n<byte><byte><byte>\r\n
helperBufferCount = 1;
Serial.println("Removed control data: 7 bytes");

return;
}

if (helperBuffer[6] == '\r' && helperBuffer[7] == '\n') {
// 8 bytes length chunk control section discovered
// \r\n<byte><byte><byte><byte>\r\n
helperBufferCount = 0;
Serial.println("Removed control data: 8 bytes");

return;
}
}

/***
* Puts a byte to the input of helper buffer and returns a byte from the output of helper buffer.
* In the meantime it tries to remove the chunk control data (if any available) from the buffer.
*
* @param incoming byte of the audio stream
* @return outgoing byte of the audio stream (if available) or -1 when no bytes available (this happens
* when the buffer is being populated with the data or after removal of chunk control data)
*/
int16_t put_through_helper_buffer(uint8_t newValue) {
//
int16_t result = -1;

if (helperBufferCount == 8) {
result = helperBuffer[0];
}

helperBuffer[0] = helperBuffer[1];
helperBuffer[1] = helperBuffer[2];
helperBuffer[2] = helperBuffer[3];
helperBuffer[3] = helperBuffer[4];
helperBuffer[4] = helperBuffer[5];
helperBuffer[5] = helperBuffer[6];
helperBuffer[6] = helperBuffer[7];
helperBuffer[7] = newValue;

helperBufferCount++;
if (helperBufferCount > 8) {
helperBufferCount = 8;
}

remove_chunk_control_data_from_helper_buffer();

return result;
}

/***
* Removes the chunk control data from the input audio stream. Data are written back to the input
* buffer and the number of bytes available (after processing) is returned.
*
* @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Transfer-Encoding
* @see https://en.wikipedia.org/wiki/Chunked_transfer_encoding
*
* @param data a pointer to the input buffer of audio stream
* @param length a number of input bytes to be processed
* @return the number of available bytes in the input buffer after removing the chunk control data
*/
uint8_t remove_chunk_control_data(uint8_t *data, size_t length) {

uint8_t writeindex = 0;
uint8_t index = 0;
for (index = 0; index < lengt; index++) {
Copy link
Owner

@baldram baldram Mar 22, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@philippedc pointed out that there is a bug: length instead of lengt.
He also said that: "by the way, if the code gives sound, the quality is awful with much more glitches than ever".
I wonder whether it's due to reducing the buffer implementation or something was changed in the code omitting glitches?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, my bad. I have fixed the typo in the lengt method signature only. Sorry.

I have no idea why it doesn;t work for @philippedc. For me it works perfect, no more glitches. I will check that again.

Copy link

@philippedc philippedc Mar 22, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is very strange...
May be your code well suppress the glitches, but the sound quality become so poor it is very brassy. May be it is a matter of performance more than a algorithm issue... by the way it is the kind of code I do not understand how it works...

I have : Arduino IDE last release 1.8.12, core ESP8266 2.4.2 and 2.6.3 (my web radio doesn't compile with 2.6.3 due to the web server library conflict)
IwIP variant : v2 higher bandwidth
CPU : 80MHz or 160MHz

Here are 2 stations that use to glitch a lot. In fact they send many chunk information....
Can you test them with your code ?
23.82.11.89:30228/stream
23.82.11.88:5128/stream

Copy link
Author

@wmarkow wmarkow Mar 23, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@philppedc,

Can you test them with your code ?
23.82.11.89:30228/stream
23.82.11.88:5128/stream

Those two URLs work nice with my code. There is no glitches at all, the sound plays nice. I have used the same settings as you:

core ESP8266 2.4.2
IwIP variant : v2 higher bandwidth
CPU : 80MHz

Does your ESP8266 also hosts a webserver inside? Maybe it has also some impact, so the sound has glitches and is brassy, because the processor also need to do the things related to the webserver?
My test code is simple; I do not have any webserver, so it only reads the stream, remove chunks data information and sends bytes to VS1053.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does your ESP8266 also hosts a webserver inside?

I've tested without and with. No special issue with.
I'm wondering if it is not a matter of cutting length: when my code cuts 5 bytes yours cuts 7 - it is what I notice on the console.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe it is because you analyze the stream in the 32 bytes chunks. When the leading \r\n is somewhere at the end of your 32 bytes chunk and the ending \r\n is somewhere at the beginning of the next 32 bytes chunk, then your algorithm may not cut it correctly.
My solution has an 8 bytes length middle buffer which is shifted constantly and the algorithm looks for those chunk contorl patterns also constantly, so I do not analyse the stream in chunks; the stream is analysed continuously.

I see, in fact if screen capture has been done with a buffer of 32 bytes, it is working identically with a buffer of 2048 bytes.
May be the glitch is due by a chunk at the high end of the buffer. But that does not answer why I find an alone \r\n in the middle of the stream. I do not have many glitches, average of one every 5 to 10 seconds so it is easy to verify the console.
However you're right with the idea of a shift buffer, I will work on it.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, so the chunk information is under the format 'number_of_bytes'\r\n. I understand that the preliminary \r\n is optional, and not count in the chunk, right ?
So I did the following:
`

int i=0;
int bytesread = BUFFERSIZE;    // = 4096
static int counter = 0;
int chunk;

if( client.available()) {
  while( i < bytesread ) {
    mp3buff[i] = client.read();

    if( i>1 && mp3buff[i] == '\n' && mp3buff[i-1] == '\r' ) {
      chunk = mp3buff[i-2];
        
      Serial.print(i); Serial.print(":"); Serial.print(chunk); Serial.print(":"); Serial.println(counter);
      counter = 0;
    }
    i++;
    counter++;
  }    // end of while

  player.playChunk(mp3buff, bytesread); 
}      // end of client.available
yield();

} // end of else
} // end of loop
`
which means that each time "I" see a \r\n sequence I read the past byte: if i=\n and i-1=\r so i-2=chunk_size.
Here is the result:
image
1st column => i
2nd column => chunk_size
3rd column => value of the byte counter
I well can see the counter indicates 5 every 2 lines: it is the count of the 5 bytes with the sequence \r\nX\r\n, but I do not understand the correlation between this X value that should be the chunk size and the result of the counter. WHat is wrong ?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

.... or does that mean when a receive a packet of 1346 bytes, only the first 48 are useful for the streaming ???

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've find the way to solve all my problems. see #52
Many thanks for your help !!!!!

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, so the chunk information is under the format 'number_of_bytes'\r\n. I understand that the preliminary \r\n is optional, and not count in the chunk, right ?

Preliminary \r\n is not optional, I think. The whole format of the stream is a s below:

metadata_about_the_stream\r\n
ZZZZ\r\n
chunk\r\n
ZZZZ\r\n
chunk\r\n
ZZZ\r\n
chunk\r\n
ZZ\r\n
chunk\r\n
Z\r\n
...

where ZZZZ or ZZZ or ZZ or Z is the length of the next chunk data. But attention, it looks like this length is encoded in ASCII HEX (human readable), so if ZZZZ is a765 then the legth of chunks is 0xa765 or if Z is 5 then the chunk length is 0x5 which is exactly five bytes. If you decode a chunk length byte as 56 DEC then according to the ASCII table it is a digit 8, so the chunk length is 8 bytes (not 56 bytes).

If you take a look again at the format above, you will notice a pattern in the stream. All what needs to be cut out is \r\nZZZZ\r\n or \r\nZZZ\r\n or \r\nZZ\r\n or \r\nZ\r\n, because these are the chunk control data; everything else what is left is an audio stream data.

uint8_t input = data[index];
int16_t output = put_through_helper_buffer(input);
if (output >= 0) {
data[writeindex] = (uint8_t) output;
writeindex++;
}
}

return writeindex;
}
2 changes: 1 addition & 1 deletion src/VS1053.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -325,4 +325,4 @@ uint16_t VS1053::getDecodedTime() {
void VS1053::clearDecodedTime() {
write_register(SCI_DECODE_TIME, 0x00);
write_register(SCI_DECODE_TIME, 0x00);
}
}
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you able to exclude this file from the patchset?
It looks like it's some accidental change.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if this is possible. I did this PR from my presonal branch and this change is there accidentaly.
I will cancel this PR and provide a new one.

Copy link
Owner

@baldram baldram Mar 22, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No need, you can just replace this line from git history or please keep it as it is, and I will exclude it while merging.