QNetworkAccessManager with Accept-Encoding gzip

[patrik08]

I subclass QNetworkAccessManager (This class was introduced in Qt 4.4.) to load remote xml html or image to compose documents.

the code is avaiable on:

http://fop-miniscribus.googlecode.co...rk/FillCache.h
http://fop-miniscribus.googlecode.co.../FillCache.cpp

I start request on this way:

void NetCacheSwap::start_Get( const QUrl url )
{
    CacheInfo havingram = take_Url(url);
    if (havingram.pending) {
    /* request ist start */
    return;
    }
    if (havingram.valid) {
    /* request and data live on swap cache */
    emit incomming(url); 
    return;
    }
    CacheInfo newrequest;
    QNetworkRequest need(url);
    need.setRawHeader( "User-Agent", "Mozilla/5.0 (X11; U; Linux i686 (x86_64); "
                       "en-US; rv:1.9.0.1) Gecko/2008070206 Firefox/3.0.1" );
    need.setRawHeader( "Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7" );
    need.setRawHeader( "Accept-Encoding", "gzip,deflate,qcompress" ); ////////gzip,deflate,qcompress
    need.setRawHeader( "Connection", "keep-alive" );
    if (cookie_host[url.host()].size() > 0) {
     qDebug() << "### > send cookie ->" << cookie_host[url.host()].toAscii();
     need.setRawHeader( "Cookie",cookie_host[url.host()].toAscii()); 
    }
    QNetworkReply *reps = get(need);
    /* mark it as pending request! */
    newrequest.pending = true;
    cache_ram.insert(fastmd5(url.toString()),newrequest);
    connect(reps, SIGNAL(downloadProgress (qint64,qint64)),
                     SLOT(progress(qint64,qint64)));
    connect(reps, SIGNAL(error(QNetworkReply::NetworkError)),
                    SLOT(url_error(QNetworkReply::NetworkError)));
    connect(reps, SIGNAL(finished()),SLOT(incomming_cache()));
}

void NetCacheSwap::start_Get( const QUrl url )
{
    CacheInfo havingram = take_Url(url);
    if (havingram.pending) {
    /* request ist start */
    return;
    }
    if (havingram.valid) {
    /* request and data live on swap cache */
    emit incomming(url); 
    return;
    }
    CacheInfo newrequest;
    QNetworkRequest need(url);
    need.setRawHeader( "User-Agent", "Mozilla/5.0 (X11; U; Linux i686 (x86_64); "
                       "en-US; rv:1.9.0.1) Gecko/2008070206 Firefox/3.0.1" );
    need.setRawHeader( "Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7" );
    need.setRawHeader( "Accept-Encoding", "gzip,deflate,qcompress" ); ////////gzip,deflate,qcompress
    need.setRawHeader( "Connection", "keep-alive" );
    if (cookie_host[url.host()].size() > 0) {
     qDebug() << "### > send cookie ->" << cookie_host[url.host()].toAscii();
     need.setRawHeader( "Cookie",cookie_host[url.host()].toAscii()); 
    }
    QNetworkReply *reps = get(need);
    /* mark it as pending request! */
    newrequest.pending = true;
    cache_ram.insert(fastmd5(url.toString()),newrequest);
    connect(reps, SIGNAL(downloadProgress (qint64,qint64)),
                     SLOT(progress(qint64,qint64)));
    connect(reps, SIGNAL(error(QNetworkReply::NetworkError)),
                    SLOT(url_error(QNetworkReply::NetworkError)));
    connect(reps, SIGNAL(finished()),SLOT(incomming_cache()));
}

To copy to clipboard, switch view to plain text mode 

if incomming chunk is and header is content-encoding == gzip i open on this way:

i decompress on file .....

QByteArray NetCacheSwap::deflate( const  QByteArray chunk )
{
    QChar letter('A' + (qrand() % 26));
    /* must not go on file solution gunzip buffer ?  go cache from net location */
    const QString tmpfiler = QString("%1/http_%2.gz").arg(location).arg(letter);
             QByteArray input;
             QFile file(tmpfiler);
             if ( file.open(QIODevice::WriteOnly) ) {
                 file.write(chunk);
                 file.close();
                 gzFile filegunzip;
                 filegunzip = gzopen(tmpfiler.toUtf8().data(),"rb");
                 if(!filegunzip) {
                 qDebug() << "### Unable to work on tmp file ... ";
                  return QByteArray();
                 }
                  char buffer[1024];
                  while(int readBytes =gzread(filegunzip, buffer, 1024))
                   {
                    input.append(QByteArray(buffer, readBytes));
                  }
                 gzclose(filegunzip);
                 file.remove();
             }
return input;          
}

QByteArray NetCacheSwap::deflate( const  QByteArray chunk )
{
    QChar letter('A' + (qrand() % 26));
    /* must not go on file solution gunzip buffer ?  go cache from net location */
    const QString tmpfiler = QString("%1/http_%2.gz").arg(location).arg(letter);
             QByteArray input;
             QFile file(tmpfiler);
             if ( file.open(QIODevice::WriteOnly) ) {
                 file.write(chunk);
                 file.close();
                 gzFile filegunzip;
                 filegunzip = gzopen(tmpfiler.toUtf8().data(),"rb");
                 if(!filegunzip) {
                 qDebug() << "### Unable to work on tmp file ... ";
                  return QByteArray();
                 }
                  char buffer[1024];
                  while(int readBytes =gzread(filegunzip, buffer, 1024))
                   {
                    input.append(QByteArray(buffer, readBytes));
                  }
                 gzclose(filegunzip);
                 file.remove();
             }
return input;          
}

To copy to clipboard, switch view to plain text mode 

How i can deflate / decompress QByteArray data direct on buffer? is this possibel?


i search "content-encoding" on network qt lib i found only

bool QHttpNetworkReplyPrivate::isGzipped()
{
QByteArray encoding = headerField("content-encoding");
return encoding.toLower() == "gzip";
}

and isGzipped search is NULL on other or webkit .. ?

[patrik08]

is this not possibel to deflate on buffer?

[DavidWoo]

Theres a solution here with code that decompresses gzip data using zlib:

http://stackoverflow.com/questions/2...ress-gzip-data

QByteArray gUncompress(const QByteArray &data)
{
    if (data.size() <= 4) {
        qWarning("gUncompress: Input data is truncated");
        return QByteArray();
    }
 
    QByteArray result;
 
    int ret;
    z_stream strm;
    static const int CHUNK_SIZE = 1024;
    char out[CHUNK_SIZE];
 
    /* allocate inflate state */
    strm.zalloc = Z_NULL;
    strm.zfree = Z_NULL;
    strm.opaque = Z_NULL;
    strm.avail_in = data.size();
    strm.next_in = (Bytef*)(data.data());
 
    ret = inflateInit2(&strm, 15 +  32); // gzip decoding
    if (ret != Z_OK)
        return QByteArray();
 
    // run inflate()
    do {
        strm.avail_out = CHUNK_SIZE;
        strm.next_out = (Bytef*)(out);
 
        ret = inflate(&strm, Z_NO_FLUSH);
        Q_ASSERT(ret != Z_STREAM_ERROR);  // state not clobbered
 
        switch (ret) {
        case Z_NEED_DICT:
            ret = Z_DATA_ERROR;     // and fall through
        case Z_DATA_ERROR:
        case Z_MEM_ERROR:
            (void)inflateEnd(&strm);
            return QByteArray();
        }
 
        result.append(out, CHUNK_SIZE - strm.avail_out);
    } while (strm.avail_out == 0);
 
    // clean up and return
    inflateEnd(&strm);
    return result;
}

QByteArray gUncompress(const QByteArray &data)
{
    if (data.size() <= 4) {
        qWarning("gUncompress: Input data is truncated");
        return QByteArray();
    }

    QByteArray result;

    int ret;
    z_stream strm;
    static const int CHUNK_SIZE = 1024;
    char out[CHUNK_SIZE];

    /* allocate inflate state */
    strm.zalloc = Z_NULL;
    strm.zfree = Z_NULL;
    strm.opaque = Z_NULL;
    strm.avail_in = data.size();
    strm.next_in = (Bytef*)(data.data());

    ret = inflateInit2(&strm, 15 +  32); // gzip decoding
    if (ret != Z_OK)
        return QByteArray();

    // run inflate()
    do {
        strm.avail_out = CHUNK_SIZE;
        strm.next_out = (Bytef*)(out);

        ret = inflate(&strm, Z_NO_FLUSH);
        Q_ASSERT(ret != Z_STREAM_ERROR);  // state not clobbered

        switch (ret) {
        case Z_NEED_DICT:
            ret = Z_DATA_ERROR;     // and fall through
        case Z_DATA_ERROR:
        case Z_MEM_ERROR:
            (void)inflateEnd(&strm);
            return QByteArray();
        }

        result.append(out, CHUNK_SIZE - strm.avail_out);
    } while (strm.avail_out == 0);

    // clean up and return
    inflateEnd(&strm);
    return result;
}

To copy to clipboard, switch view to plain text mode