[PyQt4] QTreeView performance issue

[Gad82]

Hi everyone,

I've got a QTreeView with a custom model which has to handle several items (say 1 million). I've set the setUniformRowHeights property to True, so when the tree is set up the scrolling is very fluid. The problem arises when I try to change the ordering of the items.
The following code illustrates the situation I have to deal with:



from PyQt4.QtCore import *
from PyQt4.QtGui import *
from random import shuffle
from time import clock

class TreeItem(object) :
def __init__(self, data, parent=None) :
self.parentItem = parent
self.itemData = data
self.childItems = []

def appendChild(self, item) :
self.childItems.append(item)

def child(self, row) :
return self.childItems[row]

def childCount(self) :
return len(self.childItems)

def columnCount(self) :
return len(self.itemData)

def data(self, column) :
try :
return self.itemData[column]
except IndexError :
return None

def parent(self) :
return self.parentItem

def row(self) :
if self.parentItem :
return self.parentItem.childItems.index(self)
return 0


class TreeModel(QAbstractItemModel) :
def __init__(self, ) :
QAbstractItemModel.__init__(self)

self.root = TreeItem(("root",))
self.fld = TreeItem(("Items",), self.root)
self.root.appendChild(self.fld)

self.row_id = []
for j in xrange(1000000) :
self.row_id.append(j)
self.fld.appendChild(TreeItem(("Item"+str(j+1),), self.fld))
print "Done"

def _ItemFromIndex(self, index) :
if not index.isValid() :
return self.root
item = index.internalPointer()
if item == self.fld :
return item
fld_index = self.createIndex(0, 0, self.fld)
return self.index(self.row_id[index.row()], 0, fld_index).internalPointer()

def columnCount(self, parent) :
return 1

def data(self, index, role) :
item = self._ItemFromIndex(index)
if role == Qt.DisplayRole :
return item.data(0)

def flags(self, flag) :
return (Qt.ItemIsEnabled|Qt.ItemIsUserCheckable|Qt.ItemIs Selectable)

def hasChildren(self, parent=QModelIndex()) :
if parent.column() > 0 :
return False
item = self.root if not parent.isValid() else parent.internalPointer()
return item.childCount() != 0

def index(self, row, column, parent) :
if not self.hasIndex(row, column, parent) :
return QModelIndex()

parent = self.root if not parent.isValid() else parent.internalPointer()
child = parent.child(row)
return self.createIndex(row, column, child) if child else QModelIndex()

def parent(self, index):
if not index.isValid() :
return QModelIndex()

child = index.internalPointer()
parent = child.parent()

if parent == self.root :
return QModelIndex()

return self.createIndex(parent.row(), 0, parent)

def rowCount(self, parent=QModelIndex()) :
if parent.column() > 0 :
return 0
item = self.root if not parent.isValid() else parent.internalPointer()
return item.childCount()


class Tree(QTreeView) :
def __init__(self, parent, model) :
QTreeView.__init__(self, parent)
self.setHeaderHidden(True)
self.setContextMenuPolicy(Qt.CustomContextMenu)
self.setUniformRowHeights(True)
self.setModel(model)
self.expand(self.model().index(0, 0, QModelIndex()))

def Shuffle(self) :
t1 = clock()
shuffle(self.model().row_id)
self.model().reset()
self.expand(self.model().index(0, 0, QModelIndex()))
print "Shuffle time:", clock()-t1

class Frame(QWidget) :
def __init__(self, parent, model=None) :
super(QWidget, self).__init__(parent)

self.tree = Tree(self, TreeModel())
self.shuffle_btn = QPushButton("Shuffle")
self.shuffle_btn.clicked.connect(self.tree.Shuffle )

vbox = QVBoxLayout()
vbox.setContentsMargins(0, 0, 0, 0)
vbox.addWidget(self.tree)
vbox.addWidget(self.shuffle_btn)
self.setLayout(vbox)

if __name__ == "__main__" :
import sys
a = QApplication(sys.argv)

dia = Frame(None)
dia.resize(400, 600)
dia.show()
a.exec_()


I use the row_id list to map the items to be shown at a given index; I do this in order to use my own C-written sorting function to order the items. Now if you push the “Shuffle” button (which simulates the sorting) you can see that even if the execution time is fast (0.6 sec on my pc), the tree requires several seconds to be updated.
Can anyone point me out what I am doing wrong? Thanks in advance!

[amleto]

use cprofile

http://docs.python.org/2/library/profile.html

[Gad82]

Thanks for the suggestion, I ran cProfile and got the following output:

62065498 function calls in 108.804 seconds

Ordered by: internal time

ncalls tottime percall cumtime percall filename:lineno(function)
3002777 17.933 0.000 31.866 0.000 {built-in method hasIndex}
1 14.911 14.911 39.792 39.792 {built-in method exec_}
3002777 13.876 0.000 56.146 0.000 t.py:79(index)
1 12.539 12.539 59.101 59.101 {built-in method show}
3000042 7.782 0.000 13.924 0.000 t.py:73(hasChildren)
3003126 7.145 0.000 13.124 0.000 t.py:99(rowCount)
3006293 7.037 0.000 7.037 0.000 {built-in method createIndex}
1 5.378 5.378 9.813 9.813 t.py:40(__init__)
9009969 4.122 0.000 4.122 0.000 {built-in method isValid}
1000002 3.553 0.000 3.553 0.000 t.py:7(__init__)
6003168 3.471 0.000 4.688 0.000 t.py:18(childCount)
6003168 3.186 0.000 3.186 0.000 {built-in method column}
9011961 2.267 0.000 2.267 0.000 {built-in method internalPointer}
3002777 1.238 0.000 1.238 0.000 t.py:15(child)
6003169 1.216 0.000 1.216 0.000 {len}
1 1.206 1.206 1.307 1.307 random.py:276(shuffle)
3002788 0.811 0.000 0.811 0.000 t.py:62(columnCount)
1000001 0.650 0.000 0.766 0.000 t.py:12(appendChild)
2000002 0.232 0.000 0.232 0.000 {method 'append' of 'list' objects}
999999 0.100 0.000 0.100 0.000 {method 'random' of '_random.Random' objects}
1 0.077 0.077 108.804 108.804 t.py:1(<module>)
2488 0.020 0.000 0.076 0.000 t.py:53(_ItemFromIndex)
1 0.010 0.010 0.010 0.010 {nt.urandom}
1536 0.009 0.000 0.017 0.000 t.py:87(parent)
2488 0.007 0.000 0.084 0.000 t.py:65(data)
1 0.007 0.007 0.007 0.007 {built-in method reset}
1 0.004 0.004 0.017 0.017 random.py:40(<module>)
1 0.003 0.003 0.003 0.003 t.py:107(__init__)
1 0.003 0.003 0.003 0.003 hashlib.py:55(<module>)
355 0.002 0.000 0.002 0.000 t.py:70(flags)
1122 0.001 0.000 0.002 0.000 t.py:33(row)
2394 0.001 0.000 0.001 0.000 {built-in method row}
1122 0.001 0.000 0.001 0.000 {method 'index' of 'list' objects}
1536 0.001 0.000 0.001 0.000 t.py:30(parent)
1 0.001 0.001 1.314 1.314 t.py:115(Shuffle)
1 0.000 0.000 9.817 9.817 t.py:123(__init__)
355 0.000 0.000 0.000 0.000 t.py:24(data)
1 0.000 0.000 0.000 0.000 {built-in method setModel}
4 0.000 0.000 0.000 0.000 {built-in method model}
1 0.000 0.000 0.000 0.000 {method 'connect' of 'PyQt4.QtCore.pyqtBoundSignal' o
jects}
1 0.000 0.000 0.000 0.000 {built-in method setLayout}
1 0.000 0.000 0.000 0.000 __future__.py:48(<module>)
1 0.000 0.000 0.010 0.010 random.py:100(seed)
6 0.000 0.000 0.000 0.000 hashlib.py:94(__get_openssl_constructor)
1 0.000 0.000 0.000 0.000 t.py:39(TreeModel)
1 0.000 0.000 0.000 0.000 {function seed at 0x02563CB0}
1 0.000 0.000 0.000 0.000 random.py:72(Random)
2 0.000 0.000 0.000 0.000 {built-in method expand}
1 0.000 0.000 0.000 0.000 atexit.py:6(<module>)
1 0.000 0.000 0.000 0.000 {math.exp}
2 0.000 0.000 0.000 0.000 {built-in method addWidget}
2 0.000 0.000 0.000 0.000 {time.clock}
1 0.000 0.000 0.000 0.000 {built-in method setHeaderHidden}
7 0.000 0.000 0.000 0.000 __future__.py:75(__init__)
1 0.000 0.000 0.000 0.000 {built-in method resize}
6 0.000 0.000 0.000 0.000 {getattr}
1 0.000 0.000 0.000 0.000 {built-in method setContentsMargins}
1 0.000 0.000 0.000 0.000 {binascii.hexlify}
1 0.000 0.000 0.010 0.010 random.py:91(__init__)
1 0.000 0.000 0.000 0.000 t.py:6(TreeItem)
1 0.000 0.000 0.000 0.000 {hasattr}
1 0.000 0.000 0.000 0.000 {built-in method setContextMenuPolicy}
1 0.000 0.000 0.000 0.000 atexit.py:37(register)
1 0.000 0.000 0.000 0.000 random.py:653(WichmannHill)
1 0.000 0.000 0.000 0.000 {_hashlib.openssl_md5}
1 0.000 0.000 0.000 0.000 random.py:803(SystemRandom)
1 0.000 0.000 0.000 0.000 {built-in method setUniformRowHeights}
1 0.000 0.000 0.000 0.000 {math.sqrt}
1 0.000 0.000 0.000 0.000 __future__.py:74(_Feature)
2 0.000 0.000 0.000 0.000 {math.log}
1 0.000 0.000 0.000 0.000 t.py:122(Frame)
1 0.000 0.000 0.000 0.000 t.py:106(Tree)
6 0.000 0.000 0.000 0.000 {globals}
1 0.000 0.000 0.000 0.000 {_hashlib.openssl_sha512}
1 0.000 0.000 0.000 0.000 {_hashlib.openssl_sha1}
1 0.000 0.000 0.000 0.000 {_hashlib.openssl_sha256}
1 0.000 0.000 0.000 0.000 __init__.py:1(<module>)
1 0.000 0.000 0.000 0.000 {_hashlib.openssl_sha384}
1 0.000 0.000 0.000 0.000 {_hashlib.openssl_sha224}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}


It sounds pretty strange to me...the functions hasChildren, index, hasIndex are called 3 millions (!!!) of times. As the other qt views I expect these functions to be called only for the displayed items, while here they seem to be called for all the items of the model. I really don't understand if there is an error inside my code or if I'm missing something..any ideas? Again, thanks for your help!

[norobro]

I don't think you need to reset the model in your slot.

[Gad82]

norobro, thanks for your answer. You are right, there is no need to reset the model and actually removing that line of code greatly improves performance. Nevertheless, I still have some doubts about how QTreeView handles model data. I re-ran cProfile on the script; this time I simply opened and closed the widget. Here are the most time-consuming functions:

ncalls tottime percall cumtime percall filename:lineno(function)
1 351.580 351.580 351.586 351.586 {built-in method exec_}
1 6.182 6.182 29.876 29.876 {built-in method show}
2000274 5.825 0.000 10.487 0.000 {built-in method hasIndex}
2000274 4.868 0.000 18.977 0.000 test.py:79(index)
1 3.251 3.251 6.411 6.411 test.py:40(__init__)
2000006 2.738 0.000 4.720 0.000 test.py:73(hasChildren)
1000002 2.578 0.000 2.578 0.000 test.py:7(__init__)
2000312 2.550 0.000 4.409 0.000 test.py:99(rowCount)
2000604 2.508 0.000 2.508 0.000 {built-in method createIndex}
4000318 1.232 0.000 1.597 0.000 test.py:18(childCount)
6000986 1.178 0.000 1.178 0.000 {built-in method isValid}
4000318 1.033 0.000 1.033 0.000 {built-in method column}
6001157 0.692 0.000 0.692 0.000 {built-in method internalPointer}
2000274 0.455 0.000 0.455 0.000 test.py:15(child)
1000001 0.402 0.000 0.504 0.000 test.py:12(appendChild)
4000318 0.365 0.000 0.365 0.000 {len}
2000282 0.252 0.000 0.252 0.000 test.py:62(columnCount)
2000002 0.180 0.000 0.180 0.000 {method 'append' of 'list' objects}

The functions index, hasIndex, hasChildren, rowCount...are still called 2 millions of times. As far as I know, this is not the way views are expected to work (probably that's an error in my script!). It could be really useful if someone more expert than me could give an explanation of this behaviour. Thanks!

[norobro]

You're welcome.

The following is the cProfile output that I get for your code above:

1033315 function calls in 4.485 seconds

Ordered by: internal time

ncalls tottime percall cumtime percall filename:lineno(function)
1 3.468 3.468 4.485 4.485 {built-in method exec_}
1 0.881 0.881 0.980 0.980 /usr/lib/python2.7/random.py:276(shuffle)
999999 0.099 0.000 0.099 0.000 {method 'random' of '_random.Random' objects}
3275 0.006 0.000 0.006 0.000 {built-in method createIndex}
1232 0.006 0.000 0.024 0.000 ./treeview.py:57(_ItemFromIndex)
1383 0.005 0.000 0.010 0.000 {built-in method hasIndex}
1383 0.004 0.000 0.018 0.000 ./treeview.py:83(index)
879 0.003 0.000 0.006 0.000 ./treeview.py:91(parent)
1556 0.003 0.000 0.005 0.000 ./treeview.py:103(rowCount)
1232 0.002 0.000 0.027 0.000 ./treeview.py:69(data)
5050 0.002 0.000 0.002 0.000 {built-in method isValid}
6067 0.001 0.000 0.001 0.000 {built-in method internalPointer}
179 0.001 0.000 0.001 0.000 ./treeview.py:74(flags)
1556 0.001 0.000 0.001 0.000 ./treeview.py:22(childCount)
1556 0.001 0.000 0.001 0.000 {built-in method column}
688 0.001 0.000 0.001 0.000 ./treeview.py:37(row)
1204 0.000 0.000 0.000 0.000 {built-in method row}
1383 0.000 0.000 0.000 0.000 ./treeview.py:19(child)
1557 0.000 0.000 0.000 0.000 {len}
688 0.000 0.000 0.000 0.000 {method 'index' of 'list' objects}
1383 0.000 0.000 0.000 0.000 ./treeview.py:66(columnCount)
879 0.000 0.000 0.000 0.000 ./treeview.py:34(parent)
1 0.000 0.000 0.980 0.980 ./treeview.py:119(Shuffle)
176 0.000 0.000 0.000 0.000 ./treeview.py:28(data)
2 0.000 0.000 0.000 0.000 {built-in method model}
1 0.000 0.000 0.000 0.000 {built-in method expand}
2 0.000 0.000 0.000 0.000 {time.clock}
1 0.000 0.000 4.485 4.485 <string>:1(<module>)
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
I don't know Python well but if you post (or attach) your actual code, I'll take a look.

[Gad82]

Norobro, the results I got with cProfile are obtained with the code I posted originally, not with my actual code, so it is really strange such a difference in the execution time. Maybe is there a difference in the version of PyQt? I use PyQt 4.9.5 with Python 2.7.2 under windows 7. Which is your setup?

[norobro]

I'm running Debian unstable w/ python 2.7.4 & pyqt4 4.9.3

Don't have Windows installed so I can't help you there.

The code ran slightly slower under python 2.6.8 but with the same number of calls: 1033315 function calls in 5.292 CPU seconds

[norobro]

Are you running cProfile from the command line? i.e. python -m cProfile "scriptname"
I get similar results that way.

Try running it from your script:
from PyQt4.QtCore import *
from PyQt4.QtGui import *
from random import shuffle
from time import clock
import cProfile
import pstats

class TreeItem(object) :
. . .
. . .
if __name__ == "__main__" :
import sys
a = QApplication(sys.argv)
dia = Frame(None)
dia.resize(400, 600)
dia.show()
cProfile.run('a.exec_()', 'profdata')
p = pstats.Stats('profdata')
p.sort_stats('calls').print_stats()
# a.exec_()

[Gad82]

Yes, I was running cProfile from command line; I executed the profiling as you suggested and I got values more similar to yours:



1070281 function calls in 11.630 seconds

Ordered by: call count

ncalls tottime percall cumtime percall filename:lineno(function)
999999 0.102 0.000 0.102 0.000 {method 'random' of '_random.Random' objects}
12766 0.003 0.000 0.003 0.000 {built-in method internalPointer}
10632 0.005 0.000 0.005 0.000 {built-in method isValid}
6793 0.019 0.000 0.019 0.000 {built-in method createIndex}
3388 0.001 0.000 0.001 0.000 {len}
3375 0.002 0.000 0.003 0.000 C:\Users\massimiliano\Desktop\t.py:45(childCount)
3375 0.002 0.000 0.002 0.000 {built-in method column}
3321 0.009 0.000 0.015 0.000 C:\Users\massimiliano\Desktop\t.py:126(rowCount)
2961 0.001 0.000 0.001 0.000 C:\Users\massimiliano\Desktop\t.py:42(child)
2961 0.001 0.000 0.001 0.000 C:\Users\massimiliano\Desktop\t.py:89(columnCount)
2961 0.014 0.000 0.056 0.000 C:\Users\massimiliano\Desktop\t.py:106(index)
2961 0.017 0.000 0.031 0.000 {built-in method hasIndex}
2590 0.021 0.000 0.078 0.000 C:\Users\massimiliano\Desktop\t.py:80(_ItemFromInd ex)
2590 0.008 0.000 0.086 0.000 C:\Users\massimiliano\Desktop\t.py:92(data)
2520 0.001 0.000 0.001 0.000 {built-in method row}
1706 0.001 0.000 0.001 0.000 C:\Users\massimiliano\Desktop\t.py:57(parent)
1706 0.009 0.000 0.019 0.000 C:\Users\massimiliano\Desktop\t.py:114(parent)
1312 0.002 0.000 0.003 0.000 C:\Users\massimiliano\Desktop\t.py:60(row)
1312 0.001 0.000 0.001 0.000 {method 'index' of 'list' objects}
370 0.000 0.000 0.000 0.000 C:\Users\massimiliano\Desktop\t.py:51(data)
370 0.002 0.000 0.002 0.000 C:\Users\massimiliano\Desktop\t.py:97(flags)
54 0.000 0.000 0.000 0.000 C:\Users\massimiliano\Desktop\t.py:100(hasChildren )
28 0.000 0.000 0.000 0.000 C:\Python27\lib\idlelib\rpc.py:149(debug)
20 0.000 0.000 0.000 0.000 {thread.get_ident}
20 0.000 0.000 0.000 0.000 C:\Python27\lib\threading.py:63(_note)
16 0.061 0.004 0.061 0.004 {method 'acquire' of 'thread.lock' objects}
8 0.000 0.000 0.000 0.000 {method 'release' of 'thread.lock' objects}
8 0.000 0.000 0.000 0.000 C:\Python27\lib\threading.py:58(__init__)
8 0.000 0.000 0.000 0.000 {thread.allocate_lock}
8 0.000 0.000 0.000 0.000 {isinstance}
8 0.000 0.000 0.000 0.000 C:\Python27\lib\threading.py:826(currentThread)
4 0.000 0.000 0.000 0.000 C:\Python27\lib\threading.py:141(release)
4 0.000 0.000 0.000 0.000 {getattr}
4 0.000 0.000 0.000 0.000 {method 'get' of 'dict' objects}
4 0.000 0.000 0.000 0.000 C:\Python27\lib\threading.py:186(__init__)
4 0.000 0.000 0.061 0.015 C:\Python27\lib\threading.py:235(wait)
4 0.000 0.000 0.000 0.000 C:\Python27\lib\threading.py:159(_acquire_restore)
4 0.000 0.000 0.000 0.000 C:\Python27\lib\threading.py:167(_release_save)
4 0.000 0.000 0.000 0.000 C:\Python27\lib\idlelib\rpc.py:317(newseq)
4 0.000 0.000 0.000 0.000 C:\Python27\lib\threading.py:177(_is_owned)
4 0.000 0.000 0.000 0.000 C:\Python27\lib\threading.py:101(RLock)
4 0.000 0.000 0.000 0.000 C:\Python27\lib\socket.py:223(meth)
4 0.000 0.000 0.062 0.015 C:\Python27\lib\idlelib\rpc.py:279(getresponse)
4 0.000 0.000 0.001 0.000 C:\Python27\lib\idlelib\rpc.py:218(asynccall)
4 0.000 0.000 0.062 0.015 C:\Python27\lib\idlelib\rpc.py:295(_getresponse)
4 0.000 0.000 0.000 0.000 C:\Python27\lib\threading.py:106(__init__)
4 0.000 0.000 0.000 0.000 C:\Python27\lib\idlelib\rpc.py:244(decoderesponse)
4 0.000 0.000 0.000 0.000 {select.select}
4 0.000 0.000 0.062 0.015 C:\Python27\lib\idlelib\rpc.py:238(asyncreturn)
4 0.000 0.000 0.000 0.000 {_struct.pack}
4 0.000 0.000 0.001 0.000 C:\Python27\lib\idlelib\rpc.py:321(putmessage)
4 0.000 0.000 0.063 0.016 C:\Python27\lib\idlelib\rpc.py:594(__call__)
4 0.000 0.000 0.063 0.016 C:\Python27\lib\idlelib\rpc.py:208(remotecall)
4 0.000 0.000 0.000 0.000 {cPickle.dumps}
4 0.000 0.000 0.000 0.000 {method 'fileno' of '_socket.socket' objects}
4 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects}
4 0.000 0.000 0.000 0.000 C:\Python27\lib\idlelib\rpc.py:546(__getattr__)
4 0.000 0.000 0.000 0.000 C:\Python27\lib\idlelib\rpc.py:589(__init__)
4 0.000 0.000 0.000 0.000 C:\Python27\lib\idlelib\rpc.py:287(_proxify)
4 0.000 0.000 0.000 0.000 C:\Python27\lib\threading.py:181(Condition)
4 0.001 0.000 0.001 0.000 {method 'send' of '_socket.socket' objects}
4 0.000 0.000 0.000 0.000 C:\Python27\lib\threading.py:121(acquire)
2 0.000 0.000 0.000 0.000 {time.clock}
2 0.000 0.000 0.000 0.000 {built-in method model}
1 0.003 0.003 0.003 0.003 {built-in method expand}
1 0.000 0.000 1.371 1.371 C:\Users\massimiliano\Desktop\t.py:142(Shuffle)
1 10.139 10.139 11.630 11.630 {built-in method exec_}
1 0.000 0.000 11.630 11.630 <string>:1(<module>)
1 1.203 1.203 1.305 1.305 C:\Python27\lib\random.py:276(shuffle)
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}



To be honest, I really don't understand what's going on...