Possible conversion from QStandardItemModel to QAbstractTableModel?

[jkrienert]

At the moment a QStandardItemModel is being used to support QTableView, but a future need to process large CSV files with > 50,000 rows + facilitates the need for an alternative method.

I have read several great PyQt based articles on the subject (see below) of using a QAbstractTableModel to hold the data which will then provide QtableView with items 'on demand', yet novice programing skills are making the transition anything but progressive.

http://www.saltycrane.com/blog/2007/...delqtableview/
http://sateeshkumarb.wordpress.com/2...-data-in-pyqt/
https://gist.githubusercontent.com/M...09/combobox.py

Below are the primary bits of code (currently without QAbstractTableModel implemented) called by another piece of software, in all their existing error glory.

import csv
import sys
from PyQt4 import QtGui, QtCore
from PyQt4.QtCore import Qt, QVariant
global xLoc
global yLoc
global itemIdxs
 
class CSVeditor(QtGui.QDialog, QtGui.QWidget):
    def __init__(self, fileName, horizHeaders, vertHeaders, parent=None):
        super(CSVeditor, self).__init__(parent)
 
        self.horizHeaders = horizHeaders
        self.fileName = fileName
        self.model = QtGui.QStandardItemModel(self)
        self.model.setHorizontalHeaderLabels(horizHeaders)
 
        self.tableView = QtGui.QTableView(self)
        self.tableView.setModel(self.model)
        self.tableView.horizontalHeader().setStretchLastSection(False)
        self.tableView.setAlternatingRowColors(True)
        self.tableView.hideColumn(0)
        self.tableView.hideColumn(1)
        self.tableView.hideColumn(2)
        self.tableView.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
        selectionModel = self.tableView.selectionModel()
        selectionModel.selectionChanged.connect(self.selRange)
        shortcut = QtGui.QShortcut(self)
        shortcut.setKey("Enter")
        self.connect(shortcut, QtCore.SIGNAL("activated()"), self.shareChanges)
        self.tableView.clicked.connect(self.Release)
 
        # autoLoad CSV from workspace [to be edited for dropdown CSV upload selection(s)]
        with open(fileName, "rb") as fileInput:
            for idx, row in enumerate(csv.reader(fileInput)):
                header = 0
                if idx is header:
                    pass
                elif idx>0:
                    items = [QtGui.QStandardItem(field) for field in row]
                    self.model.appendRow(items)
        self.model.setVerticalHeaderLabels(vertHeaders)
        self.show()
 
    # index of selected range, and parent item (last index) for batch application of user-value changes
    def selRange(self,selected,deselected):
        selectedIdx = self.tableView.selectedIndexes()
        idxCol = []
        idxRow = []
        for row in selectedIdx:
            idxRow.append(row.row())
        for column in selectedIdx:
            idxCol.append(column.column())
        global itemIdxs
        itemIdxs = zip(idxRow,idxCol)
 
    def shareChanges(self):
        self.tableView.clicked.connect(self.Release)
        try:
            updatedVal = self.model.data(self.model.index(xLoc,yLoc+3))
            print xLoc,yLoc,updatedVal
        except AttributeError:
            pass
        readOnlyCols = [0]
        try:
            for i in itemIdxs:
                row = i[0]
                col = i[1]
                if col in readOnlyCols:
                    pass
                else:
                    self.model.setData(self.model.index(row,col), updatedVal, Qt.DisplayRole)
        except AttributeError:
            print ":("
            pass
 
    # button induced save-change to CSV (NO UNDO YET!)
    def writeCsv(self, fileName):
        with open(fileName, "wb") as fileOutput:
            writer = csv.writer(fileOutput)
            writer.writerow(self.horizHeaders)
            for rowNumber in range(self.model.rowCount()):
                fields = [self.model.data(self.model.index(rowNumber,columnNumber),QtCore.Qt.DisplayRole) for columnNumber in range(self.model.columnCount())]
                writer.writerow(fields)
            print ':)'
 
    @QtCore.pyqtSlot()
    def on_pushButtonWrite_clicked(self):
        self.writeCsv(self.fileName)
 
    def Cancel(self):
        self.close()
 
    def closeEvent(self,event):
        self.deleteLater()
 
    def Release(self,index):
        global xLoc
        global yLoc
        xLoc,yLoc = index.row(),(index.column()-3)

import csv
import sys
from PyQt4 import QtGui, QtCore
from PyQt4.QtCore import Qt, QVariant
global xLoc
global yLoc
global itemIdxs

class CSVeditor(QtGui.QDialog, QtGui.QWidget):
    def __init__(self, fileName, horizHeaders, vertHeaders, parent=None):
        super(CSVeditor, self).__init__(parent)

        self.horizHeaders = horizHeaders
        self.fileName = fileName
        self.model = QtGui.QStandardItemModel(self)
        self.model.setHorizontalHeaderLabels(horizHeaders)

        self.tableView = QtGui.QTableView(self)
        self.tableView.setModel(self.model)
        self.tableView.horizontalHeader().setStretchLastSection(False)
        self.tableView.setAlternatingRowColors(True)
        self.tableView.hideColumn(0)
        self.tableView.hideColumn(1)
        self.tableView.hideColumn(2)
        self.tableView.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
        selectionModel = self.tableView.selectionModel()
        selectionModel.selectionChanged.connect(self.selRange)
        shortcut = QtGui.QShortcut(self)
        shortcut.setKey("Enter")
        self.connect(shortcut, QtCore.SIGNAL("activated()"), self.shareChanges)
        self.tableView.clicked.connect(self.Release)

        # autoLoad CSV from workspace [to be edited for dropdown CSV upload selection(s)]
        with open(fileName, "rb") as fileInput:
            for idx, row in enumerate(csv.reader(fileInput)):
                header = 0
                if idx is header:
                    pass
                elif idx>0:
                    items = [QtGui.QStandardItem(field) for field in row]
                    self.model.appendRow(items)
        self.model.setVerticalHeaderLabels(vertHeaders)
        self.show()

    # index of selected range, and parent item (last index) for batch application of user-value changes
    def selRange(self,selected,deselected):
        selectedIdx = self.tableView.selectedIndexes()
        idxCol = []
        idxRow = []
        for row in selectedIdx:
            idxRow.append(row.row())
        for column in selectedIdx:
            idxCol.append(column.column())
        global itemIdxs
        itemIdxs = zip(idxRow,idxCol)

    def shareChanges(self):
        self.tableView.clicked.connect(self.Release)
        try:
            updatedVal = self.model.data(self.model.index(xLoc,yLoc+3))
            print xLoc,yLoc,updatedVal
        except AttributeError:
            pass
        readOnlyCols = [0]
        try:
            for i in itemIdxs:
                row = i[0]
                col = i[1]
                if col in readOnlyCols:
                    pass
                else:
                    self.model.setData(self.model.index(row,col), updatedVal, Qt.DisplayRole)
        except AttributeError:
            print ":("
            pass

    # button induced save-change to CSV (NO UNDO YET!)
    def writeCsv(self, fileName):
        with open(fileName, "wb") as fileOutput:
            writer = csv.writer(fileOutput)
            writer.writerow(self.horizHeaders)
            for rowNumber in range(self.model.rowCount()):
                fields = [self.model.data(self.model.index(rowNumber,columnNumber),QtCore.Qt.DisplayRole) for columnNumber in range(self.model.columnCount())]
                writer.writerow(fields)
            print ':)'

    @QtCore.pyqtSlot()
    def on_pushButtonWrite_clicked(self):
        self.writeCsv(self.fileName)

    def Cancel(self):
        self.close()

    def closeEvent(self,event):
        self.deleteLater()

    def Release(self,index):
        global xLoc
        global yLoc
        xLoc,yLoc = index.row(),(index.column()-3)

To copy to clipboard, switch view to plain text mode 

Is it feasible to chop shop existing, or start from scratch rather?

[anda_skoa]

As a first step you could make a table model that holds all the data, e.g. in a 2-dimensional array.
That should allow you to basically use the same loading code you are currently using.

Cheers,
_

[jkrienert]

Thanks for the quickness anda_skoa
In this manner (?) :

class CSVModel(QtCore.QAbstractTableModel):
    def __init__(self, CSVdata, fileName, parent = None):
        QAbstractTableModel.__init__(self, parent)
        self.CSVreader.rows = CSVdata
    def rowCount(self, parent):
        return len(self.CSVreader.rows)
    def columnCount(self, parent):
        return len(self.CSVreader.header)
    def data(self, index, role):
        self.CSVreader.rows[index.row()][index.column()]
    def CSVreader(self,fileName):
        header = []
        rows = []
        with open(fileName, "rb") as fileInput:
            for idx, row in enumerate(csv.reader(fileInput)):
                headIDx = 0
                if idx is headIDx:
                    header.append(row)
                elif idx>headIDx:
                    items = [field for field in row]
                    rows.append(items)

class CSVModel(QtCore.QAbstractTableModel):
    def __init__(self, CSVdata, fileName, parent = None):
        QAbstractTableModel.__init__(self, parent)
        self.CSVreader.rows = CSVdata
    def rowCount(self, parent):
        return len(self.CSVreader.rows)
    def columnCount(self, parent):
        return len(self.CSVreader.header)
    def data(self, index, role):
        self.CSVreader.rows[index.row()][index.column()]
    def CSVreader(self,fileName):
        header = []
        rows = []
        with open(fileName, "rb") as fileInput:
            for idx, row in enumerate(csv.reader(fileInput)):
                headIDx = 0
                if idx is headIDx:
                    header.append(row)
                elif idx>headIDx:
                    items = [field for field in row]
                    rows.append(items)

To copy to clipboard, switch view to plain text mode 

[d_stranz]

but a future need to process large CSV files with > 50,000 rows + facilitates the need for an alternative method.

Constructing QStandardItem instances for every cell in a table is a lot of overhead, so mapping your data from an internal data structure into a table model instead of using QStandardItemModel would be good.

And as anda_skoa said, for the first step, simply implement the table model to hold the entire set of data. If this indeed kills performance, then go to plan B. Don't try to optimize before you know you have to.