NoRulez
9th April 2008, 10:25
Hey @all,
now comes my first post ;-) This forum is really great.
My Problem is to Transfer large binary files over the network >= 90 GB. So i will do this as fast as possible.
My client and server are clones from the fortune client and threaded fortune server.
Can anybody explain me or give me a hint what's wrong. Because when i transfer a file from windows to windows the server prints "readClient();" but the client is already finished. And on the other situation when i transfer from Windows to Linux, the client dies with the message: "QWaitCondition: Destroyed while threads are still waiting" and the server wrote something with "Can't load type 4321342 from QVariant".
But here is the Client code and in the next posting is the server-thread code:
main.cpp
#include <QtCore/QCoreApplication>
#include "client.h"
#include <iostream>
using namespace std;
int main(int argc, char *argv[]) {
QCoreApplication a(argc, argv);
Client client(a.argv()[1], a.argv()[2]);
client.transfer_file(QString("%1").arg(a.argv()[3]).toInt(), QString("%1").arg(a.argv()[4]).toInt(),
a.argv()[5], a.argv()[6]);
return a.exec();
}
client.h
#ifndef CLIENT_H
#define CLIENT_H
#include <QObject>
#include <QTcpSocket>
#include <QSslSocket>
#include <QSslError>
#include <QFile>
#include <QFileInfo>
#include <iostream>
#include <windows.h>
class Client : public QObject {
Q_OBJECT
public:
Client(QString hostname, QString port);//(QObject *parent);
~Client();
void setDir(QString _directory);
private slots:
void onStateChanged(QAbstractSocket::SocketState socketState);
void onMessage(const QString &msg, const QVariant &data);
void onConnected();
void onDisconnected();
void sendMessage(const QString &msg, const QVariant &data);
void onReadyRead();
void displayError(QAbstractSocket::SocketError socketError);
/*void socketEncrypted();
void sslErrors(const QList<QSslError> &errors);
void onEncrypted();*/
public slots:
void transfer_file(qint64 _bytes_to_read, qint64 _to_sleep = 50, QString filename = "", QString remote_filename = "");
//void transfer_file(QString filename);
private:
double tick_diff;
qint64 max_bytes_for_all;
QString _directory;
QTcpSocket *tcpSocket;
//QSslSocket *tcpSSLSocket;
QString currentFortune;
quint32 blockSize;
bool reading_message;
qint64 percentage(qint64 max, qint64 work);
};
#endif // CLIENT_H
client.cpp
#include "client.h"
#include <QVariant>
#include <iostream>
#include <QDir>
#include <QSslError>
#include <QSslSocket>
#include <QSslKey>
#include <QSslCipher>
#include "messages.h"
using namespace std;
#define BUFFER_SIZE 2*1024*1024 // 2 MB 16384 // 16 kb
#define TRANSFER_SIZE 16384*3 //2*1024*1024 // 2 MB 16384 // 16 kb
#define TEST_LOOP 1000000
LARGE_INTEGER start_ticks, ende_ticks, frequenz;
Client::Client(QString hostname, QString port) {//(QObject *parent) : QObject(parent) {
tcpSocket = new QTcpSocket(this);
connect(tcpSocket, SIGNAL(readyRead()), this, SLOT(onReadyRead()));
connect(tcpSocket, SIGNAL(error(QAbstractSocket::SocketError)), this, SLOT(displayError(QAbstractSocket::SocketError)));
connect(tcpSocket, SIGNAL(disconnected()), this, SLOT(onDisconnected()));
connect(tcpSocket, SIGNAL(connected()), this, SLOT(onConnected()));
connect(tcpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)) , this, SLOT(onStateChanged(QAbstractSocket::SocketState)) );
blockSize = 0;
tick_diff = 0;
max_bytes_for_all = 0;
blockSize = 0;
tcpSocket->abort();
tcpSocket->connectToHost(hostname, port.toInt());
const int Timeout = 5 * 1000;
if (!tcpSocket->waitForConnected(Timeout)) {
emit displayError(tcpSocket->error());//, tcpSocket->errorString());
//QMessageBox::critical(this, "SOCKET ERROR", tcpSocket->errorString());
return;
}
}
Client::~Client() {
}
void Client::onStateChanged(QAbstractSocket::SocketStat e socketState) {
}
void Client::setDir(QString _directory) {
this->_directory = _directory;
}
void Client::onMessage( const QString &msg, const QVariant &data ) {
if(msg == GD_RETR_WELCOME_STRING) {
if(data.type() == QVariant::Map) {
QMap<QString, QVariant> val;
val = data.toMap();
for(QMap<QString, QVariant>::iterator it = val.begin();
it != val.end(); ++it) {
qDebug() << it.key() << ": " << it.value() << endl;
}
}
}
else {
if(data.type() == QVariant::Map) {
QMap<QString, QVariant> val;
val = data.toMap();
for(QMap<QString, QVariant>::iterator it = val.begin();
it != val.end(); ++it) {
qDebug() << it.key() << ": " << it.value() << endl;
}
}
}
}
void Client::onConnected() {
}
void Client::onDisconnected() {
if(tcpSocket == NULL)
return;
tcpSocket->abort();
}
void Client::sendMessage(const QString &msg, const QVariant &data) {
if(tcpSocket == NULL) return;
if(tcpSocket->state() != QAbstractSocket::ConnectedState ) return;
QByteArray block;
QDataStream out(&block, QIODevice::WriteOnly);
out.setVersion(QDataStream::Qt_4_2);
//out.setVersion( out.version() ); // set to the current Qt version
out << (quint32) 0;
//out << (quint16) 0;
out << msg;
out << data;
out.device()->seek(0);
out << (quint32)(block.size() - sizeof(quint32));
//out << (quint16)(block.size() - sizeof(quint16));
tcpSocket->write(block);
tcpSocket->flush();
}
void Client::onReadyRead() {
if (tcpSocket == NULL) return;
if (tcpSocket->state() != QAbstractSocket::ConnectedState) return;
QDataStream in(tcpSocket);
in.setVersion(QDataStream::Qt_4_2);
//in.setVersion( in.version() ); // set to the current Qt version instead
if(blockSize == 0) {
if (tcpSocket->bytesAvailable() < (int)sizeof(quint32)) return;
//if (tcpSocket->bytesAvailable() < (int)sizeof(quint16)) return;
in >> blockSize;
reading_message = true;
}
if(tcpSocket->bytesAvailable() < blockSize) return;
QString msgString;
QVariant msgData;
in >> msgString;
in >> msgData;
emit onMessage(msgString, msgData);
reading_message = false;
blockSize = 0;
if(tcpSocket->bytesAvailable() > 0) onReadyRead();
}
void Client::displayError(QAbstractSocket::SocketError socketError) {
switch (socketError) {
case QAbstractSocket::RemoteHostClosedError:
break;
case QAbstractSocket::HostNotFoundError:
cerr << "The host was not found. Please check the host name and port settings." << endl;
break;
case QAbstractSocket::ConnectionRefusedError:
cerr << "The connection was refused by the peer." << endl;
cerr << "Make sure the fortune server is running, " << endl;
cerr << "and check that the host name and port settings are correct." << endl;
break;
default:
cerr << QString("The following error occurred: %1.").
arg(tcpSocket->errorString()).toStdString() << endl;
}
exit(socketError + 1);
}
qint64 Client::percentage(qint64 max, qint64 work) {
return (work*100/max);
}
void Client::transfer_file(qint64 _bytes_to_read, qint64 _to_sleep, QString filename, QString remote_filename) {
QFileInfo fi(filename);
qint64 bytes_to_read = _bytes_to_read;//1048576;//2*1024*102416*1024;//1*1024;//16*1024;
qint64 bytes_read = 0;
qint64 max_bytes = fi.size();
qint64 full_max_bytes = max_bytes;
qint64 read_bytes = 0;
qint64 br = 0;
cout << "Filesize: " << fi.size() << endl;
cout << "Filesize: " << max_bytes << endl;
cout << "Filesize: " << full_max_bytes << endl;
// return;
QFile file(filename);
if(!file.open(QIODevice::ReadOnly))
cout << "Can't read from file '" << filename.toStdString() << "'" << endl;
while (max_bytes != 0) {
//while (numberOfPackages > 0){
QMap<QString, QVariant> new_val;
new_val.insert("MD5SUM", "");
new_val.insert("CHUNK", bytes_read);
new_val.insert("FILENAME", filename);
new_val.insert("FILENAME_NEW", remote_filename);
//QCryptographicHash::Md5
QByteArray byteArray;
if(bytes_to_read > full_max_bytes)
bytes_to_read = full_max_bytes;
read_bytes = bytes_to_read;
if((read_bytes + bytes_read) > full_max_bytes)
read_bytes = full_max_bytes - bytes_to_read;
byteArray = file.read(read_bytes);
max_bytes -= byteArray.size();//br;
new_val.insert("SIZE", byteArray.size());
if(tcpSocket->state()!=QAbstractSocket::ConnectedState){
printf("Socket disconnected error\n");
break;
}
if (tcpSocket->bytesToWrite() > 0 && !tcpSocket->waitForBytesWritten(10000)){
printf("Socket write error\n");
printf("socket error: %d %s, socket state: %d\n", tcpSocket->error(), tcpSocket->errorString().toAscii().data(),
tcpSocket->state());
break;
}
new_val.insert("CONTENT", byteArray);
//emit sendMessage(GD_FILE, new_val);
sendMessage(GD_FILE, new_val);
bytes_read += byteArray.size();//br;
cout << percentage(full_max_bytes, bytes_read) << " % " << filename.toStdString() << "\r";
cout.flush();
_sleep(_to_sleep);
}
file.close();
}
now comes my first post ;-) This forum is really great.
My Problem is to Transfer large binary files over the network >= 90 GB. So i will do this as fast as possible.
My client and server are clones from the fortune client and threaded fortune server.
Can anybody explain me or give me a hint what's wrong. Because when i transfer a file from windows to windows the server prints "readClient();" but the client is already finished. And on the other situation when i transfer from Windows to Linux, the client dies with the message: "QWaitCondition: Destroyed while threads are still waiting" and the server wrote something with "Can't load type 4321342 from QVariant".
But here is the Client code and in the next posting is the server-thread code:
main.cpp
#include <QtCore/QCoreApplication>
#include "client.h"
#include <iostream>
using namespace std;
int main(int argc, char *argv[]) {
QCoreApplication a(argc, argv);
Client client(a.argv()[1], a.argv()[2]);
client.transfer_file(QString("%1").arg(a.argv()[3]).toInt(), QString("%1").arg(a.argv()[4]).toInt(),
a.argv()[5], a.argv()[6]);
return a.exec();
}
client.h
#ifndef CLIENT_H
#define CLIENT_H
#include <QObject>
#include <QTcpSocket>
#include <QSslSocket>
#include <QSslError>
#include <QFile>
#include <QFileInfo>
#include <iostream>
#include <windows.h>
class Client : public QObject {
Q_OBJECT
public:
Client(QString hostname, QString port);//(QObject *parent);
~Client();
void setDir(QString _directory);
private slots:
void onStateChanged(QAbstractSocket::SocketState socketState);
void onMessage(const QString &msg, const QVariant &data);
void onConnected();
void onDisconnected();
void sendMessage(const QString &msg, const QVariant &data);
void onReadyRead();
void displayError(QAbstractSocket::SocketError socketError);
/*void socketEncrypted();
void sslErrors(const QList<QSslError> &errors);
void onEncrypted();*/
public slots:
void transfer_file(qint64 _bytes_to_read, qint64 _to_sleep = 50, QString filename = "", QString remote_filename = "");
//void transfer_file(QString filename);
private:
double tick_diff;
qint64 max_bytes_for_all;
QString _directory;
QTcpSocket *tcpSocket;
//QSslSocket *tcpSSLSocket;
QString currentFortune;
quint32 blockSize;
bool reading_message;
qint64 percentage(qint64 max, qint64 work);
};
#endif // CLIENT_H
client.cpp
#include "client.h"
#include <QVariant>
#include <iostream>
#include <QDir>
#include <QSslError>
#include <QSslSocket>
#include <QSslKey>
#include <QSslCipher>
#include "messages.h"
using namespace std;
#define BUFFER_SIZE 2*1024*1024 // 2 MB 16384 // 16 kb
#define TRANSFER_SIZE 16384*3 //2*1024*1024 // 2 MB 16384 // 16 kb
#define TEST_LOOP 1000000
LARGE_INTEGER start_ticks, ende_ticks, frequenz;
Client::Client(QString hostname, QString port) {//(QObject *parent) : QObject(parent) {
tcpSocket = new QTcpSocket(this);
connect(tcpSocket, SIGNAL(readyRead()), this, SLOT(onReadyRead()));
connect(tcpSocket, SIGNAL(error(QAbstractSocket::SocketError)), this, SLOT(displayError(QAbstractSocket::SocketError)));
connect(tcpSocket, SIGNAL(disconnected()), this, SLOT(onDisconnected()));
connect(tcpSocket, SIGNAL(connected()), this, SLOT(onConnected()));
connect(tcpSocket, SIGNAL(stateChanged(QAbstractSocket::SocketState)) , this, SLOT(onStateChanged(QAbstractSocket::SocketState)) );
blockSize = 0;
tick_diff = 0;
max_bytes_for_all = 0;
blockSize = 0;
tcpSocket->abort();
tcpSocket->connectToHost(hostname, port.toInt());
const int Timeout = 5 * 1000;
if (!tcpSocket->waitForConnected(Timeout)) {
emit displayError(tcpSocket->error());//, tcpSocket->errorString());
//QMessageBox::critical(this, "SOCKET ERROR", tcpSocket->errorString());
return;
}
}
Client::~Client() {
}
void Client::onStateChanged(QAbstractSocket::SocketStat e socketState) {
}
void Client::setDir(QString _directory) {
this->_directory = _directory;
}
void Client::onMessage( const QString &msg, const QVariant &data ) {
if(msg == GD_RETR_WELCOME_STRING) {
if(data.type() == QVariant::Map) {
QMap<QString, QVariant> val;
val = data.toMap();
for(QMap<QString, QVariant>::iterator it = val.begin();
it != val.end(); ++it) {
qDebug() << it.key() << ": " << it.value() << endl;
}
}
}
else {
if(data.type() == QVariant::Map) {
QMap<QString, QVariant> val;
val = data.toMap();
for(QMap<QString, QVariant>::iterator it = val.begin();
it != val.end(); ++it) {
qDebug() << it.key() << ": " << it.value() << endl;
}
}
}
}
void Client::onConnected() {
}
void Client::onDisconnected() {
if(tcpSocket == NULL)
return;
tcpSocket->abort();
}
void Client::sendMessage(const QString &msg, const QVariant &data) {
if(tcpSocket == NULL) return;
if(tcpSocket->state() != QAbstractSocket::ConnectedState ) return;
QByteArray block;
QDataStream out(&block, QIODevice::WriteOnly);
out.setVersion(QDataStream::Qt_4_2);
//out.setVersion( out.version() ); // set to the current Qt version
out << (quint32) 0;
//out << (quint16) 0;
out << msg;
out << data;
out.device()->seek(0);
out << (quint32)(block.size() - sizeof(quint32));
//out << (quint16)(block.size() - sizeof(quint16));
tcpSocket->write(block);
tcpSocket->flush();
}
void Client::onReadyRead() {
if (tcpSocket == NULL) return;
if (tcpSocket->state() != QAbstractSocket::ConnectedState) return;
QDataStream in(tcpSocket);
in.setVersion(QDataStream::Qt_4_2);
//in.setVersion( in.version() ); // set to the current Qt version instead
if(blockSize == 0) {
if (tcpSocket->bytesAvailable() < (int)sizeof(quint32)) return;
//if (tcpSocket->bytesAvailable() < (int)sizeof(quint16)) return;
in >> blockSize;
reading_message = true;
}
if(tcpSocket->bytesAvailable() < blockSize) return;
QString msgString;
QVariant msgData;
in >> msgString;
in >> msgData;
emit onMessage(msgString, msgData);
reading_message = false;
blockSize = 0;
if(tcpSocket->bytesAvailable() > 0) onReadyRead();
}
void Client::displayError(QAbstractSocket::SocketError socketError) {
switch (socketError) {
case QAbstractSocket::RemoteHostClosedError:
break;
case QAbstractSocket::HostNotFoundError:
cerr << "The host was not found. Please check the host name and port settings." << endl;
break;
case QAbstractSocket::ConnectionRefusedError:
cerr << "The connection was refused by the peer." << endl;
cerr << "Make sure the fortune server is running, " << endl;
cerr << "and check that the host name and port settings are correct." << endl;
break;
default:
cerr << QString("The following error occurred: %1.").
arg(tcpSocket->errorString()).toStdString() << endl;
}
exit(socketError + 1);
}
qint64 Client::percentage(qint64 max, qint64 work) {
return (work*100/max);
}
void Client::transfer_file(qint64 _bytes_to_read, qint64 _to_sleep, QString filename, QString remote_filename) {
QFileInfo fi(filename);
qint64 bytes_to_read = _bytes_to_read;//1048576;//2*1024*102416*1024;//1*1024;//16*1024;
qint64 bytes_read = 0;
qint64 max_bytes = fi.size();
qint64 full_max_bytes = max_bytes;
qint64 read_bytes = 0;
qint64 br = 0;
cout << "Filesize: " << fi.size() << endl;
cout << "Filesize: " << max_bytes << endl;
cout << "Filesize: " << full_max_bytes << endl;
// return;
QFile file(filename);
if(!file.open(QIODevice::ReadOnly))
cout << "Can't read from file '" << filename.toStdString() << "'" << endl;
while (max_bytes != 0) {
//while (numberOfPackages > 0){
QMap<QString, QVariant> new_val;
new_val.insert("MD5SUM", "");
new_val.insert("CHUNK", bytes_read);
new_val.insert("FILENAME", filename);
new_val.insert("FILENAME_NEW", remote_filename);
//QCryptographicHash::Md5
QByteArray byteArray;
if(bytes_to_read > full_max_bytes)
bytes_to_read = full_max_bytes;
read_bytes = bytes_to_read;
if((read_bytes + bytes_read) > full_max_bytes)
read_bytes = full_max_bytes - bytes_to_read;
byteArray = file.read(read_bytes);
max_bytes -= byteArray.size();//br;
new_val.insert("SIZE", byteArray.size());
if(tcpSocket->state()!=QAbstractSocket::ConnectedState){
printf("Socket disconnected error\n");
break;
}
if (tcpSocket->bytesToWrite() > 0 && !tcpSocket->waitForBytesWritten(10000)){
printf("Socket write error\n");
printf("socket error: %d %s, socket state: %d\n", tcpSocket->error(), tcpSocket->errorString().toAscii().data(),
tcpSocket->state());
break;
}
new_val.insert("CONTENT", byteArray);
//emit sendMessage(GD_FILE, new_val);
sendMessage(GD_FILE, new_val);
bytes_read += byteArray.size();//br;
cout << percentage(full_max_bytes, bytes_read) << " % " << filename.toStdString() << "\r";
cout.flush();
_sleep(_to_sleep);
}
file.close();
}