QSharedMemory::handle: doesn't exist

[pymab]

Hi,

I'm currently experimenting with QSharedMemory and QProcess in PyQt. So I wrote a small application to launch a process. From this process I create a shared memory segment and write data to it. The application reads the data back when the process writes on the output. Unfortunately I get this error when the application attempts to attach to the shared memory segment: QSharedMemory::handle: doesn't exist.

Is it possible to do this sort of things ? Or maybe I should use a different approach to achieve this ?

Thanks for the help!
-mab

My application code:

from PyQt4 import QtGui, QtCore
import sys
import pickle
 
class Widget(QtGui.QWidget):
    def __init__(self):
        super(Widget,self).__init__()
 
        # create process
        self.p = QtCore.QProcess(self)    
 
        # Connect to process output 
        self.p.readyReadStandardOutput.connect( self.on_process_output )
 
        self.key = 'share_mem_key'
        self.shmem = QtCore.QSharedMemory( key )
 
        self.p.start( 'python.exe test_process.py "%s"' % self.key )
 
    def on_process_output(self):
        self.shmem = QtCore.QSharedMemory( self.key )
        if not self.shmem.isAttached() and not self.shmem.attach():
            # Error when trying to attach to the shared memory segment
            print 'ERROR %s ' % self.shmem.errorString()
 
        self.shmem.lock()
        try:
            print pickle.loads(self.shmem.data().asstring())
        finally:
            self.shmem.unlock()
 
# application loop    
app = QtGui.QApplication(sys.argv)
widget = Widget()
 
widget.show()
app.exec_()

from PyQt4 import QtGui, QtCore
import sys
import pickle

class Widget(QtGui.QWidget):
    def __init__(self):
        super(Widget,self).__init__()
        
        # create process
        self.p = QtCore.QProcess(self)    
        
        # Connect to process output 
        self.p.readyReadStandardOutput.connect( self.on_process_output )
    
        self.key = 'share_mem_key'
        self.shmem = QtCore.QSharedMemory( key )

        self.p.start( 'python.exe test_process.py "%s"' % self.key )
                               
    def on_process_output(self):
        self.shmem = QtCore.QSharedMemory( self.key )
        if not self.shmem.isAttached() and not self.shmem.attach():
            # Error when trying to attach to the shared memory segment
            print 'ERROR %s ' % self.shmem.errorString()
        
        self.shmem.lock()
        try:
            print pickle.loads(self.shmem.data().asstring())
        finally:
            self.shmem.unlock()
                         
# application loop    
app = QtGui.QApplication(sys.argv)
widget = Widget()

widget.show()
app.exec_()

To copy to clipboard, switch view to plain text mode 

The external process code:

from PyQt4 import QtCore
import ctypes
import ctypes.util
import pickle
import sys
 
CLIB = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c'))
 
def main(argv):   
    key = argv[1]
 
    # write to shared memory      
    data = range(50000)
    data_bytes = pickle.dumps( data, pickle.HIGHEST_PROTOCOL)
    data_len = len(data_bytes)      
 
    shmem = QtCore.QSharedMemory(key)
    if not shmem.create(data_len):
        sys.stderr.write( 'ERROR: shared memory creation' )
        sys.stderr.flush()        
        return
 
    if not shmem.isAttached() and not shmem.attach():
        sys.stderr.write( 'ERROR: shared memory access' )
        sys.stderr.flush()        
        return 
 
    shmem.lock()
    try:
        CLIB.memcpy(int(shmem.data()), data_bytes, data_len)
    finally:
        shmem.unlock()
 
    sys.stdout.write( 'done' )
    sys.stdout.flush()
 
if __name__ == '__main__':
    main(sys.argv)

from PyQt4 import QtCore
import ctypes
import ctypes.util
import pickle
import sys

CLIB = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c'))

def main(argv):   
    key = argv[1]

    # write to shared memory      
    data = range(50000)
    data_bytes = pickle.dumps( data, pickle.HIGHEST_PROTOCOL)
    data_len = len(data_bytes)      

    shmem = QtCore.QSharedMemory(key)
    if not shmem.create(data_len):
        sys.stderr.write( 'ERROR: shared memory creation' )
        sys.stderr.flush()        
        return
    
    if not shmem.isAttached() and not shmem.attach():
        sys.stderr.write( 'ERROR: shared memory access' )
        sys.stderr.flush()        
        return 

    shmem.lock()
    try:
        CLIB.memcpy(int(shmem.data()), data_bytes, data_len)
    finally:
        shmem.unlock()
     
    sys.stdout.write( 'done' )
    sys.stdout.flush()
   
if __name__ == '__main__':
    main(sys.argv)

To copy to clipboard, switch view to plain text mode 

[wysota]

Does the shared memory block exist when the parent process tries to access it? Have you verified the child process successfully creates it? Which platform you are using, Windows? I don't see that Qt returns such a message for Windows anywhere, only for UNIX.

[pymab]

Does the shared memory block exist when the parent process tries to access it?

Apparently no, I get this error when trying to access it from the output callback:
: QSharedMemory::handle: doesn't exist

Have you verified the child process successfully creates it?

Yes, everything is ok since I don't get an error msg in the process error callback.

Which platform you are using, Windows? I don't see that Qt returns such a message for Windows anywhere, only for UNIX.

I'm on Windows. Here's the output trace (from my updated code below):
<pre>
process state: Starting
process state: Running
process started
process output: Done writing to shared memory
Error accessing shared memory from parent process QSharedMemory::handle: doesn't exist
Error accessing data
process state: Not running
process finished
</pre>

App:

from PyQt4 import QtGui, QtCore
import sys
import pickle
 
class Widget(QtGui.QWidget):
    def __init__(self):
        super(Widget,self).__init__()
 
        # create process
        self.p = QtCore.QProcess(self)    
 
        # Connect to process output 
        self.p.readyReadStandardOutput.connect( self.on_process_output )
        self.p.readyReadStandardError.connect( self.on_process_error_output )
        self.p.stateChanged.connect(self.on_process_state_change)                   
        self.p.finished.connect(self.on_process_finished)        
        self.p.started.connect(self.on_process_started)
        self.p.error.connect( self.on_process_error )
 
        self.key = 'share_mem_key'
        self.shmem = QtCore.QSharedMemory( self.key )
 
        self.p.start( 'python.exe shmem_process_test.py "%s"' % self.key )
 
    def on_process_output(self):        
        s_out = bytes.decode( bytes( self.sender().readAllStandardOutput() ) )        
        print 'process output: %s' % (s_out)
 
        if not self.shmem.isAttached() and not self.shmem.attach():
            print 'Error accessing shared memory from parent process: %s ' % self.shmem.errorString()
 
        self.shmem.lock()
        try:
            data = self.shmem.data()
            if data:                
                print pickle.loads(data.asstring())
        finally:
            print 'Error accessing data'
            self.shmem.unlock()
 
    def on_process_error_output(self):
        s_out = bytes.decode( bytes( self.sender().readAllStandardError() ) )                
        print 'process output: %s' % (s_out)
 
    def on_process_state_change(self,new_state):        
        states = ["Not running", "Starting", "Running"]        
        print 'process state: %s' % (states[new_state])
 
    def on_process_finished(self):
        print 'process finished'
 
    def on_process_started(self):
        print 'process started'
 
    def on_process_error(self):
        print 'process error'
 
# application loop    
app = QtGui.QApplication(sys.argv)
widget = Widget()
 
widget.show()
app.exec_()

from PyQt4 import QtGui, QtCore
import sys
import pickle
 
class Widget(QtGui.QWidget):
    def __init__(self):
        super(Widget,self).__init__()
 
        # create process
        self.p = QtCore.QProcess(self)    
        
        # Connect to process output 
        self.p.readyReadStandardOutput.connect( self.on_process_output )
        self.p.readyReadStandardError.connect( self.on_process_error_output )
        self.p.stateChanged.connect(self.on_process_state_change)                   
        self.p.finished.connect(self.on_process_finished)        
        self.p.started.connect(self.on_process_started)
        self.p.error.connect( self.on_process_error )

        self.key = 'share_mem_key'
        self.shmem = QtCore.QSharedMemory( self.key )
        
        self.p.start( 'python.exe shmem_process_test.py "%s"' % self.key )
         
    def on_process_output(self):        
        s_out = bytes.decode( bytes( self.sender().readAllStandardOutput() ) )        
        print 'process output: %s' % (s_out)

        if not self.shmem.isAttached() and not self.shmem.attach():
            print 'Error accessing shared memory from parent process: %s ' % self.shmem.errorString()
         
        self.shmem.lock()
        try:
            data = self.shmem.data()
            if data:                
                print pickle.loads(data.asstring())
        finally:
            print 'Error accessing data'
            self.shmem.unlock()
        
    def on_process_error_output(self):
        s_out = bytes.decode( bytes( self.sender().readAllStandardError() ) )                
        print 'process output: %s' % (s_out)

    def on_process_state_change(self,new_state):        
        states = ["Not running", "Starting", "Running"]        
        print 'process state: %s' % (states[new_state])

    def on_process_finished(self):
        print 'process finished'

    def on_process_started(self):
        print 'process started'

    def on_process_error(self):
        print 'process error'
        
# application loop    
app = QtGui.QApplication(sys.argv)
widget = Widget()
 
widget.show()
app.exec_()

To copy to clipboard, switch view to plain text mode 

Process:

from PyQt4 import QtCore
import ctypes
import ctypes.util
import pickle
import sys
 
CLIB = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c'))
 
def main(argv):   
    key = argv[1]
 
    # write to shared memory      
    data = range(50)
    data_bytes = pickle.dumps( data, pickle.HIGHEST_PROTOCOL)
    data_len = len(data_bytes)      
 
    shmem = QtCore.QSharedMemory(key)
 
    if not shmem.create(data_len):
        sys.stderr.write( 'ERROR: shared memory creation' )
        sys.stderr.flush()        
        return
 
    if not shmem.isAttached() and not shmem.attach():
        sys.stderr.write( 'ERROR: shared memory access' )
        sys.stderr.flush()        
        return 
 
    shmem.lock()
    try:
        CLIB.memcpy(int(shmem.data()), data_bytes, data_len)
    finally:
        shmem.unlock()
 
    sys.stdout.write( "Done writing to shared memory" )
    sys.stdout.flush()
 
if __name__ == '__main__':
    main(sys.argv)

from PyQt4 import QtCore
import ctypes
import ctypes.util
import pickle
import sys
 
CLIB = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c'))
 
def main(argv):   
    key = argv[1]
 
    # write to shared memory      
    data = range(50)
    data_bytes = pickle.dumps( data, pickle.HIGHEST_PROTOCOL)
    data_len = len(data_bytes)      

    shmem = QtCore.QSharedMemory(key)

    if not shmem.create(data_len):
        sys.stderr.write( 'ERROR: shared memory creation' )
        sys.stderr.flush()        
        return

    if not shmem.isAttached() and not shmem.attach():
        sys.stderr.write( 'ERROR: shared memory access' )
        sys.stderr.flush()        
        return 

    shmem.lock()
    try:
        CLIB.memcpy(int(shmem.data()), data_bytes, data_len)
    finally:
        shmem.unlock()
 
    sys.stdout.write( "Done writing to shared memory" )
    sys.stdout.flush()
 
if __name__ == '__main__':
    main(sys.argv)

To copy to clipboard, switch view to plain text mode 

So it seems like the only way the parent can successfully access the shared memory is if the memory is allocated by itself before starting the child process. This is not what I want. What I want is to let the child process allocate the memory with the right size instead of allocating upfront in the parent process.

Thanks for looking at this.
-mab