Upload files bigger than available memory

[knobtviker]

Can someone please explain to me how to upload files which are bigger than available memory?
I looked at every example and they all load complete files with readAll() and then send them.
This is not an option for me on S60 platform coz I hit Out-of-memory error.
I tried to do it with QNetworkAccessManager:: post() method and I failed.
Now, I'm trying to use QTcpSocket, but I don't know how to send and format the header before sending content??
I attached a part of my code:

tcpSocket = new QTcpSocket();
 
QString param = "upload.txt";
 
QNetworkAccessManager *http_upload = new QNetworkAccessManager(this);
QNetworkRequest requpl;
 
QString toDropboxDir = metadata.value("path");
if (!toDropboxDir.isEmpty())
{
        toDropboxDir = toDropboxDir.right(toDropboxDir.size()-1);
}
 
std::string stuff_std = OAuthWebRequestUpload("http://api-content.dropbox.com/0/files/dropbox/"+toDropboxDir.toStdString(),"POST",consumerKey,consumerSecret,token,secret,"",param.toStdString());
QString stuff = QString::fromStdString(stuff_std);
QByteArray stuff_BA;
stuff_BA = stuff.toAscii();
 
qsrand(QDateTime::currentDateTime().toTime_t());    
QString boundary_random = QVariant(qrand()).toString()+QVariant(qrand()).toString()+QVariant(qrand()).toString();  
QString crlf("\r\n");
QString boundaryStr("---------------------------"+boundary_random);
QString boundary="--"+boundaryStr+crlf;
 
    tcpSocket->connectToHost("api-content.dropbox.com",80);
    connect(tcpSocket, SIGNAL(connected()), this, SLOT(filler()));
    connect(tcpSocket, SIGNAL(error(QAbstractSocket::SocketError)), this, SLOT(erase_multipartform()));
 
//setting header to the request (with neccessary OAuth authorization)
    requpl.setUrl(QUrl(QString("http://api-content.dropbox.com/0/files/dropbox/"+metadata.value("path"))+"?file="+param));
    requpl.setRawHeader("Authorization", stuff_BA);
    requpl.setHeader(QNetworkRequest::ContentTypeHeader, "multipart/form-data; boundary="+boundaryStr);
    QFile *file_upload = new QFile(param_path);
    file_upload->open(QIODevice::ReadOnly);
 
//sending content
    QByteArray payload;
    QDataStream stream(&payload, QIODevice::WriteOnly);
    stream.setVersion(QDataStream::Qt_4_7);
    stream << "Authorization "+stuff;
        qDebug() << "Authorization "+stuff;
    stream << "--"+boundaryStr << "\r\n";
        //qDebug() << "--"+boundaryStr << "\r\n";
    stream << "Content-Disposition: form-data; name=\"file\"; filename=\""+param+"\"" << "\r\n";
        //qDebug() << "Content-Disposition: form-data; name=\"file\"; filename=\""+param+"\"" << "\r\n";
    stream << "Content-Type: text/plain" << "\r\n" << "\r\n";
        //qDebug() << "Content-Type: "+param_mime << "\r\n" << "\r\n";
 
    int iBlock = 0;
    while(!file_upload->atEnd())
    {
        qDebug()<<"Reading Line: "<<++iBlock;
        //stream << (quint16)0;
        //stream << file_upload->readLine().data();
        stream << file_upload->readAll().data();
        tcpSocket->write(payload);
        tcpSocket->flush();
            //qDebug() << QString(file_upload->readAll().constData());
        payload.clear();
 
    }
    file_upload->close();
    stream << "\r\n" << "--"+boundaryStr+"--" << "\r\n";
        //qDebug() << "\r\n" << "--"+boundaryStr+"--" << "\r\n";
    reply_upload = http_upload->post(requpl,payload);
    connect(reply_upload, SIGNAL(finished()), this, SLOT(readData_upload()));

tcpSocket = new QTcpSocket();

QString param = "upload.txt";

QNetworkAccessManager *http_upload = new QNetworkAccessManager(this);
QNetworkRequest requpl;

QString toDropboxDir = metadata.value("path");
if (!toDropboxDir.isEmpty())
{
        toDropboxDir = toDropboxDir.right(toDropboxDir.size()-1);
}

std::string stuff_std = OAuthWebRequestUpload("http://api-content.dropbox.com/0/files/dropbox/"+toDropboxDir.toStdString(),"POST",consumerKey,consumerSecret,token,secret,"",param.toStdString());
QString stuff = QString::fromStdString(stuff_std);
QByteArray stuff_BA;
stuff_BA = stuff.toAscii();

qsrand(QDateTime::currentDateTime().toTime_t());    
QString boundary_random = QVariant(qrand()).toString()+QVariant(qrand()).toString()+QVariant(qrand()).toString();  
QString crlf("\r\n");
QString boundaryStr("---------------------------"+boundary_random);
QString boundary="--"+boundaryStr+crlf;

    tcpSocket->connectToHost("api-content.dropbox.com",80);
    connect(tcpSocket, SIGNAL(connected()), this, SLOT(filler()));
    connect(tcpSocket, SIGNAL(error(QAbstractSocket::SocketError)), this, SLOT(erase_multipartform()));

//setting header to the request (with neccessary OAuth authorization)
    requpl.setUrl(QUrl(QString("http://api-content.dropbox.com/0/files/dropbox/"+metadata.value("path"))+"?file="+param));
    requpl.setRawHeader("Authorization", stuff_BA);
    requpl.setHeader(QNetworkRequest::ContentTypeHeader, "multipart/form-data; boundary="+boundaryStr);
    QFile *file_upload = new QFile(param_path);
    file_upload->open(QIODevice::ReadOnly);

//sending content
    QByteArray payload;
    QDataStream stream(&payload, QIODevice::WriteOnly);
    stream.setVersion(QDataStream::Qt_4_7);
    stream << "Authorization "+stuff;
        qDebug() << "Authorization "+stuff;
    stream << "--"+boundaryStr << "\r\n";
        //qDebug() << "--"+boundaryStr << "\r\n";
    stream << "Content-Disposition: form-data; name=\"file\"; filename=\""+param+"\"" << "\r\n";
        //qDebug() << "Content-Disposition: form-data; name=\"file\"; filename=\""+param+"\"" << "\r\n";
    stream << "Content-Type: text/plain" << "\r\n" << "\r\n";
        //qDebug() << "Content-Type: "+param_mime << "\r\n" << "\r\n";

    int iBlock = 0;
    while(!file_upload->atEnd())
    {
        qDebug()<<"Reading Line: "<<++iBlock;
        //stream << (quint16)0;
        //stream << file_upload->readLine().data();
        stream << file_upload->readAll().data();
        tcpSocket->write(payload);
        tcpSocket->flush();
            //qDebug() << QString(file_upload->readAll().constData());
        payload.clear();

    }
    file_upload->close();
    stream << "\r\n" << "--"+boundaryStr+"--" << "\r\n";
        //qDebug() << "\r\n" << "--"+boundaryStr+"--" << "\r\n";
    reply_upload = http_upload->post(requpl,payload);
    connect(reply_upload, SIGNAL(finished()), this, SLOT(readData_upload()));

To copy to clipboard, switch view to plain text mode 

Please help me, I'm stuck on this for days now...

[wysota]

Did you try the QNetworkAccessManager::post() that takes a QIODevice as a parameter?

[knobtviker]

Hi, thanks for answering.
Yes I tried it with that method but then I can't load the boundarys like that and that implies (if I understand the docs properly) that the file still loads up entirely which is fine for small files but not for the large ones.

I rewrote my code, went low level on this one and learned a lot. But it's still not working, somehow something is either wrong with my generated POST form or the TCP socket is not connected properly.
Take a look and tell me what I'm missing out:

//TESTING PARAMETERS START
    QString param = "upload.txt";
    //QString param = "Zastitaokolisa.rar";
 
    QString param_mime = "text/plain";
    //QString param_mime = "application/x-rar-compressed";
 
    QString param_path = "E://Dropian//"+param;
    //QString param_path = "E://"+param;
    //TESTING PARAMETERS END
 
    //SETUP TCPSOCKET AND DATA
    QString toDropboxDir = metadata.value("path");
    if (!toDropboxDir.isEmpty())
    {
        toDropboxDir = toDropboxDir.right(toDropboxDir.size()-1);
    }
 
    std::string stuff_std = OAuthWebRequestUpload("http://api-  content.dropbox.com/0/files/dropbox/"+toDropboxDir.toStdString(),"POST",consumerKey,consumerSecret,token,secret,"",param.toStdString());
    QString stuff = QString::fromStdString(stuff_std);
    QByteArray stuff_BA;
    stuff_BA = stuff.toAscii();
 
    qsrand(QDateTime::currentDateTime().toTime_t());    
    QString boundary_random = QVariant(qrand()).toString()+QVariant(qrand()).toString()+QVariant(qrand()).toString();  
    QString crlf("\r\n");
    QString boundaryStr("---------------------------"+boundary_random);
    QString boundary="--"+boundaryStr+crlf;
 
    tcpSocket = new QTcpSocket();
 
    tcpSocket->connectToHost("api-content.dropbox.com",80);
    connect(tcpSocket, SIGNAL(connected()), this, SLOT(filler()));
    connect(tcpSocket, SIGNAL(error(QAbstractSocket::SocketError)), this, SLOT(erase_multipartform()));
 
 
    QFile *file_upload = new QFile(param_path);
    file_upload->open(QIODevice::ReadOnly);    
 
    QString header;
    header += "POST /0/files/dropbox/"+metadata.value("path")+" HTTP/1.1\r\n";
    header += "Host: api-content.dropbox.com\r\n";
    header += "Content-Type: multipart/form-data; boundary="+boundaryStr+"\r\n";
    header += "Authorization: "+stuff+"\r\n\r\n";
 
    QString content;
    content += "--"+boundaryStr+"\r\n";
    content += "Content-Disposition: form-data; name=\"file\"; filename=\""+param+"\"\r\n";
    content += "Content-Type: "+param_mime+"\r\n\r\n";
 
    QString end;
    end += "\r\n--"+boundaryStr+"--\r\n";
 
    QByteArray payload;
    QDataStream stream(&payload, QIODevice::WriteOnly);
    stream.setVersion(QDataStream::Qt_4_7);
 
    //HEADER STREAM START
    stream << header;
        qDebug() << header;
    //tcpSocket->waitForConnected(30000);
    tcpSocket->write(payload);
    tcpSocket->flush();
    payload.clear();
    header.clear();
    //HEADER STREAM END
 
    //CONTENT STREAM START
    stream << content;
        qDebug() << content;
    //tcpSocket->waitForConnected(30000);
    tcpSocket->write(payload);
    tcpSocket->flush();
    payload.clear();
    content.clear();
 
    //UPLOADING FILE START
    int iBlock = 0;
    while(!file_upload->atEnd())
    {
        qDebug()<<"Reading Line: "<<++iBlock;
        //stream << (quint16)0;
        stream << file_upload->readLine().data();
        //stream << file_upload->readAll().data();
        //tcpSocket->waitForConnected(30000);
        tcpSocket->write(payload);
        tcpSocket->flush();
            //qDebug() << QString(file_upload->readLine().constData());
        //stream.device()->seek(0);
        //stream << (quint16)(payload.size()-sizeof(quint16));
        payload.clear();
 
    }
    file_upload->close();
    //UPLOADING FILE END
 
    stream << end;
        qDebug() << end;
    //tcpSocket->waitForConnected(30000);
    tcpSocket->write(payload);
    tcpSocket->flush();
    payload.clear();
    end.clear();
    //CONTENT STREAM END
 
    //tcpSocket->waitForConnected(30000);
    tcpSocket->disconnect();

//TESTING PARAMETERS START
    QString param = "upload.txt";
    //QString param = "Zastitaokolisa.rar";

    QString param_mime = "text/plain";
    //QString param_mime = "application/x-rar-compressed";

    QString param_path = "E://Dropian//"+param;
    //QString param_path = "E://"+param;
    //TESTING PARAMETERS END

    //SETUP TCPSOCKET AND DATA
    QString toDropboxDir = metadata.value("path");
    if (!toDropboxDir.isEmpty())
    {
        toDropboxDir = toDropboxDir.right(toDropboxDir.size()-1);
    }

    std::string stuff_std = OAuthWebRequestUpload("http://api-  content.dropbox.com/0/files/dropbox/"+toDropboxDir.toStdString(),"POST",consumerKey,consumerSecret,token,secret,"",param.toStdString());
    QString stuff = QString::fromStdString(stuff_std);
    QByteArray stuff_BA;
    stuff_BA = stuff.toAscii();

    qsrand(QDateTime::currentDateTime().toTime_t());    
    QString boundary_random = QVariant(qrand()).toString()+QVariant(qrand()).toString()+QVariant(qrand()).toString();  
    QString crlf("\r\n");
    QString boundaryStr("---------------------------"+boundary_random);
    QString boundary="--"+boundaryStr+crlf;

    tcpSocket = new QTcpSocket();

    tcpSocket->connectToHost("api-content.dropbox.com",80);
    connect(tcpSocket, SIGNAL(connected()), this, SLOT(filler()));
    connect(tcpSocket, SIGNAL(error(QAbstractSocket::SocketError)), this, SLOT(erase_multipartform()));


    QFile *file_upload = new QFile(param_path);
    file_upload->open(QIODevice::ReadOnly);    

    QString header;
    header += "POST /0/files/dropbox/"+metadata.value("path")+" HTTP/1.1\r\n";
    header += "Host: api-content.dropbox.com\r\n";
    header += "Content-Type: multipart/form-data; boundary="+boundaryStr+"\r\n";
    header += "Authorization: "+stuff+"\r\n\r\n";

    QString content;
    content += "--"+boundaryStr+"\r\n";
    content += "Content-Disposition: form-data; name=\"file\"; filename=\""+param+"\"\r\n";
    content += "Content-Type: "+param_mime+"\r\n\r\n";

    QString end;
    end += "\r\n--"+boundaryStr+"--\r\n";

    QByteArray payload;
    QDataStream stream(&payload, QIODevice::WriteOnly);
    stream.setVersion(QDataStream::Qt_4_7);

    //HEADER STREAM START
    stream << header;
        qDebug() << header;
    //tcpSocket->waitForConnected(30000);
    tcpSocket->write(payload);
    tcpSocket->flush();
    payload.clear();
    header.clear();
    //HEADER STREAM END

    //CONTENT STREAM START
    stream << content;
        qDebug() << content;
    //tcpSocket->waitForConnected(30000);
    tcpSocket->write(payload);
    tcpSocket->flush();
    payload.clear();
    content.clear();

    //UPLOADING FILE START
    int iBlock = 0;
    while(!file_upload->atEnd())
    {
        qDebug()<<"Reading Line: "<<++iBlock;
        //stream << (quint16)0;
        stream << file_upload->readLine().data();
        //stream << file_upload->readAll().data();
        //tcpSocket->waitForConnected(30000);
        tcpSocket->write(payload);
        tcpSocket->flush();
            //qDebug() << QString(file_upload->readLine().constData());
        //stream.device()->seek(0);
        //stream << (quint16)(payload.size()-sizeof(quint16));
        payload.clear();

    }
    file_upload->close();
    //UPLOADING FILE END

    stream << end;
        qDebug() << end;
    //tcpSocket->waitForConnected(30000);
    tcpSocket->write(payload);
    tcpSocket->flush();
    payload.clear();
    end.clear();
    //CONTENT STREAM END

    //tcpSocket->waitForConnected(30000);
    tcpSocket->disconnect();

To copy to clipboard, switch view to plain text mode 

And here is the POST form it generates:

POST /0/files/dropbox/ HTTP/1.1
Host: api-content.dropbox.com
Content-Type: multipart/form-data; boundary=---------------------------63101371611732759261053641528
Authorization: OAuth file=upload.txt,oauth_consumer_key=1234567890abcdef,oauth_nonce=Kqm88U9yhT7XfeL0,oauth_signature=WTXVsyETru0GR%2BiK5ZxW9MNuGG8%3D,oauth_signature_method=HMAC-SHA1,oauth_timestamp=1292846464,oauth_token=1234567890abcdef,oauth_version=1.0
 
-----------------------------63101371611732759261053641528
Content-Disposition: form-data; name="file"; filename="upload.txt"
Content-Type: text/plain
 
Hello world! 
 
-----------------------------63101371611732759261053641528--

POST /0/files/dropbox/ HTTP/1.1
Host: api-content.dropbox.com
Content-Type: multipart/form-data; boundary=---------------------------63101371611732759261053641528
Authorization: OAuth file=upload.txt,oauth_consumer_key=1234567890abcdef,oauth_nonce=Kqm88U9yhT7XfeL0,oauth_signature=WTXVsyETru0GR%2BiK5ZxW9MNuGG8%3D,oauth_signature_method=HMAC-SHA1,oauth_timestamp=1292846464,oauth_token=1234567890abcdef,oauth_version=1.0

-----------------------------63101371611732759261053641528
Content-Disposition: form-data; name="file"; filename="upload.txt"
Content-Type: text/plain

Hello world! 

-----------------------------63101371611732759261053641528--

To copy to clipboard, switch view to plain text mode 

I admit, I'm a n00b at QTcpSocket so don't laugh too hard at this.
Thanks for your help in advance.

[Raketten]

Hi all,

did you ever find a solution to this problem?

Rgds Henrik

[ChrisW67]

Did you read Wysota's post?