Revision history [back]

Hey Guys, Long story short I am trying to extract results into excel quicker and am trying to multi-thread this process. However it seems that even though the multi-thread side of things works within my script only 1 file will properly extract and the rest will just be empty workbooks.

 Process Process-54:
Traceback (most recent call last):
  File "C:\Python27\lib\multiprocessing\process.py", line 267, in _bootstrap
    self.run()
  File "C:\Python27\lib\multiprocessing\process.py", line 114, in run
    self._target(*self._args, **self._kwargs)
  File "C:\Dropbox\Power System Services\Scripts\New folder\PYTHON\2.7\PSSE Python Scripts\PSSE\PSSE_MAT\DynDataExtractorMATMulti.py", line 179, in run_data_extract
    outfile=outfile1, sheet='', overwritesheet=True);
  File ".\dyntools.py", line 1084, in xlsout
  File ".\dyntools.py", line 428, in _xls_export
  File ".\excelpy.py", line 168, in <module>
  File "C:\Python27\lib\site-packages\win32com\client\__init__.py", line 11, in <module>
    import gencache
  File "C:\Python27\lib\site-packages\win32com\client\gencache.py", line 662, in <module>
    __init__()
  File "C:\Python27\lib\site-packages\win32com\client\gencache.py", line 56, in __init__
    _LoadDicts()
  File "C:\Python27\lib\site-packages\win32com\client\gencache.py", line 109, in _LoadDicts
    version = p.load()
EOFError

This is the error code that comes out. This is essentially the only line of the code which is causing these errors.

dyntools.CHNF(outfile1).xlsout(channels=signals, show=False, xlsfile=(PathDir + "\\sheets\\" + filename + str(fileNumber) + '.xlsx'),
                 outfile='', sheet='', overwritesheet=True);

Note this code works as intended if I reduce the thread count to 1 however that sort of defeats the point. Cheers

Hey Guys, Long story short I am trying to extract results into excel quicker and am trying to multi-thread this process. However it seems that even though the multi-thread side of things works within my script only 1 file will properly extract and the rest will just be empty workbooks.

 Process Process-54:
Traceback (most recent call last):
  File "C:\Python27\lib\multiprocessing\process.py", line 267, in _bootstrap
    self.run()
  File "C:\Python27\lib\multiprocessing\process.py", line 114, in run
    self._target(*self._args, **self._kwargs)
  File "C:\Dropbox\Power System Services\Scripts\New folder\PYTHON\2.7\PSSE Python Scripts\PSSE\PSSE_MAT\DynDataExtractorMATMulti.py", line 179, in run_data_extract
    outfile=outfile1, sheet='', overwritesheet=True);
  File ".\dyntools.py", line 1084, in xlsout
  File ".\dyntools.py", line 428, in _xls_export
  File ".\excelpy.py", line 168, in <module>
  File "C:\Python27\lib\site-packages\win32com\client\__init__.py", line 11, in <module>
    import gencache
  File "C:\Python27\lib\site-packages\win32com\client\gencache.py", line 662, in <module>
    __init__()
  File "C:\Python27\lib\site-packages\win32com\client\gencache.py", line 56, in __init__
    _LoadDicts()
  File "C:\Python27\lib\site-packages\win32com\client\gencache.py", line 109, in _LoadDicts
    version = p.load()
EOFError

This is the error code that comes out. This is essentially the only line of the code which is causing these errors.

dyntools.CHNF(outfile1).xlsout(channels=signals, show=False, xlsfile=(PathDir + "\\sheets\\" + filename + str(fileNumber) + '.xlsx'),
                 outfile='', sheet='', overwritesheet=True);

Note this code works as intended if I reduce the thread count to 1 however that sort of defeats the point. Cheers

Traceback (most recent call last):
  File "C:/Dropbox/Power System Services/Scripts/New folder/PYTHON/2.7/PSSE Python Scripts/PSSE/PSSE_MAT/ExtractionPSSE.py", line 137, in <module>
    main()
  File "C:/Dropbox/Power System Services/Scripts/New folder/PYTHON/2.7/PSSE Python Scripts/PSSE/PSSE_MAT/ExtractionPSSE.py", line 115, in main
    results = out_pool.map(build_xls, processes)
  File "C:\Python27\lib\multiprocessing\pool.py", line 253, in map
    return self.map_async(func, iterable, chunksize).get()
  File "C:\Python27\lib\multiprocessing\pool.py", line 572, in get
    raise self._value
pywintypes.com_error: (-2147418111, 'Call was rejected by callee.', None, None)

import sys
import os
import shutil
import Tkinter, tkFileDialog
import ntpath
import csv
import time
import multiprocessing
from multiprocessing import Pool as ThreadPool
import math
PSSE33BINPATH = r"C:/Program Files (x86)/PTI/PSSEUniversity33/pssbin;C:/Program Files (x86)/PTI/PSSEUniversity33/psslib;"

def main():

    # Prompt user for input

    root = Tkinter.Tk()
    root.withdraw()

    resultType = "Teebar"

    dirname = tkFileDialog.askopenfilename(parent=root, initialdir="/",
                                        title='Please select a _0.out File')
    PathDir = os.path.dirname(dirname)
    PathDir = os.path.abspath(os.path.join(PathDir, os.pardir))

    if not os.path.exists(PathDir + r"\graphs"):
        os.makedirs(PathDir + r"\graphs")
    else:
        shutil.rmtree(PathDir + r"\graphs")
        os.makedirs(PathDir + r"\graphs")
    if not os.path.exists(PathDir + r"\logs"):
        os.makedirs(PathDir + r"\logs")
    else:
        shutil.rmtree(PathDir + r"\logs")
        os.makedirs(PathDir + r"\logs")
    if not os.path.exists(PathDir + r"\sheets"):
        os.makedirs(PathDir + r"\sheets")
    else:
        shutil.rmtree(PathDir + r"\sheets")
        os.makedirs(PathDir + r"\sheets")
    dirname=str(dirname.replace("/","\\"))
    dirname=str(dirname.replace("0.out",""))
    filename = ntpath.basename(dirname)

    # determine number of cases
    check=True
    k=0
    while check is True:
        my_file =(dirname+ str(k) + '.out')
        if os.path.isfile(my_file):
            k=k+1
        else:
            print('number is :'+ str(k))
            check = False

    signals = ['1','2','3','4']


    processes = [] # setup blank list for processes
    # number_of_threads = int(math.ceil(multiprocessing.cpu_count()))

    number_of_threads = 18 #10core / 20 Thread Processor (this wouldn't work even if set to 2)


    # Loop through out files to setup an argument list
    for i in range(0,k):
        print(i)
        # file_number += 1 # arbitrary
        out = dirname + str(i)
        # Check if xlsx exists and delete
        my_file = out + '.xlsx'
        if os.path.isfile(my_file):
            os.remove(my_file)

        # Current Output File name
        out_file = out + '.out'
        print(out_file)
        arguments = (out_file, signals, PathDir, filename, i)
        processes.append(arguments)

    # Here I set number_of_threads to equal processes if I don't need to use all available threads
    if len(processes) < number_of_threads:
        number_of_threads = len(processes)

    # Build pool and map your processes to it
    out_pool = ThreadPool(number_of_threads)
    results = out_pool.map(build_xls, processes)
    out_pool.close()
    out_pool.join()  # Wait until all threads are finished


def build_xls((outfile, signals, path_dir, file_name, file_number)):
    # The following may be different depending on PSSE version
    os.environ['PATH'] = PSSE33BINPATH + ';' + os.environ['PATH']
    sys.path.append('C:/Program Files (x86)/PTI/PSSEUniversity33/pssbin')
    sys.path.append('C:/Program Files (x86)/PTI/PSSEUniversity33/psslib')

    # sys.path.insert(0, PSSE33BINPATH)

    import dyntools
    # print(outfile)
    excel_file = path_dir + "\\sheets\\" + file_name + str(file_number) + '.xlsx'
    achnf = dyntools.CHNF(outfile);
    achnf.xlsout(channels=signals, show=False, xlsfile=excel_file, outfile='', sheet ='', overwritesheet=True);
    # dyntools.CHNF(outfile).xlsout(channels=signals, show=False, xlsfile=excel_file, outfile='', sheet='', overwritesheet=True)
    time.sleep(20)

if __name__=='__main__':
    main()

Hey Guys, Long story short I am trying to extract results into excel quicker and am trying to multi-thread this process. However it seems that even though the multi-thread side of things works within my script only 1 file will properly extract and the rest will just be empty workbooks.

 Process Process-54:
Traceback (most recent call last):
  File "C:\Python27\lib\multiprocessing\process.py", line 267, in _bootstrap
    self.run()
  File "C:\Python27\lib\multiprocessing\process.py", line 114, in run
    self._target(*self._args, **self._kwargs)
  File "C:\Dropbox\Power System Services\Scripts\New folder\PYTHON\2.7\PSSE Python Scripts\PSSE\PSSE_MAT\DynDataExtractorMATMulti.py", line 179, in run_data_extract
    outfile=outfile1, sheet='', overwritesheet=True);
  File ".\dyntools.py", line 1084, in xlsout
  File ".\dyntools.py", line 428, in _xls_export
  File ".\excelpy.py", line 168, in <module>
  File "C:\Python27\lib\site-packages\win32com\client\__init__.py", line 11, in <module>
    import gencache
  File "C:\Python27\lib\site-packages\win32com\client\gencache.py", line 662, in <module>
    __init__()
  File "C:\Python27\lib\site-packages\win32com\client\gencache.py", line 56, in __init__
    _LoadDicts()
  File "C:\Python27\lib\site-packages\win32com\client\gencache.py", line 109, in _LoadDicts
    version = p.load()
EOFError

This is the error code that comes out. This is essentially the only line of the code which is causing these errors.

dyntools.CHNF(outfile1).xlsout(channels=signals, show=False, xlsfile=(PathDir + "\\sheets\\" + filename + str(fileNumber) + '.xlsx'),
                 outfile='', sheet='', overwritesheet=True);

Note this code works as intended if I reduce the thread count to 1 however that sort of defeats the point. Cheers

EDIT Thanks for the response EBahr. - I implemented your method of multi-threading the process and still recieving an error once I increase from 1 thread to any other number.

Traceback (most recent call last):
  File "C:/Dropbox/Power System Services/Scripts/New folder/PYTHON/2.7/PSSE Python Scripts/PSSE/PSSE_MAT/ExtractionPSSE.py", line 137, in <module>
    main()
  File "C:/Dropbox/Power System Services/Scripts/New folder/PYTHON/2.7/PSSE Python Scripts/PSSE/PSSE_MAT/ExtractionPSSE.py", line 115, in main
    results = out_pool.map(build_xls, processes)
  File "C:\Python27\lib\multiprocessing\pool.py", line 253, in map
    return self.map_async(func, iterable, chunksize).get()
  File "C:\Python27\lib\multiprocessing\pool.py", line 572, in get
    raise self._value
pywintypes.com_error: (-2147418111, 'Call was rejected by callee.', None, None)

Below is my implementation it simply requires a set of Out files named like so. (Please excuse my lack of structure :/) Results_0.out

Results_1.out

Results_2.out

Results_3.out

import sys
import os
import shutil
import Tkinter, tkFileDialog
import ntpath
import csv
import time
import multiprocessing
from multiprocessing import Pool as ThreadPool
import math
PSSE33BINPATH = r"C:/Program Files (x86)/PTI/PSSEUniversity33/pssbin;C:/Program Files (x86)/PTI/PSSEUniversity33/psslib;"

def main():

    # Prompt user for input

    root = Tkinter.Tk()
    root.withdraw()

    resultType = "Teebar"

    dirname = tkFileDialog.askopenfilename(parent=root, initialdir="/",
                                        title='Please select a _0.out File')
    PathDir = os.path.dirname(dirname)
    PathDir = os.path.abspath(os.path.join(PathDir, os.pardir))

    if not os.path.exists(PathDir + r"\graphs"):
        os.makedirs(PathDir + r"\graphs")
    else:
        shutil.rmtree(PathDir + r"\graphs")
        os.makedirs(PathDir + r"\graphs")
    if not os.path.exists(PathDir + r"\logs"):
        os.makedirs(PathDir + r"\logs")
    else:
        shutil.rmtree(PathDir + r"\logs")
        os.makedirs(PathDir + r"\logs")
    if not os.path.exists(PathDir + r"\sheets"):
        os.makedirs(PathDir + r"\sheets")
    else:
        shutil.rmtree(PathDir + r"\sheets")
        os.makedirs(PathDir + r"\sheets")
    dirname=str(dirname.replace("/","\\"))
    dirname=str(dirname.replace("0.out",""))
    filename = ntpath.basename(dirname)

    # determine number of cases
    check=True
    k=0
    while check is True:
        my_file =(dirname+ str(k) + '.out')
        if os.path.isfile(my_file):
            k=k+1
        else:
            print('number is :'+ str(k))
            check = False

    signals = ['1','2','3','4']


    processes = [] # setup blank list for processes
    # number_of_threads = int(math.ceil(multiprocessing.cpu_count()))

    number_of_threads = 18 #10core / 20 Thread Processor (this wouldn't work even if set to 2)


    # Loop through out files to setup an argument list
    for i in range(0,k):
        print(i)
        # file_number += 1 # arbitrary
        out = dirname + str(i)
        # Check if xlsx exists and delete
        my_file = out + '.xlsx'
        if os.path.isfile(my_file):
            os.remove(my_file)

        # Current Output File name
        out_file = out + '.out'
        print(out_file)
        arguments = (out_file, signals, PathDir, filename, i)
        processes.append(arguments)

    # Here I set number_of_threads to equal processes if I don't need to use all available threads
    if len(processes) < number_of_threads:
        number_of_threads = len(processes)

    # Build pool and map your processes to it
    out_pool = ThreadPool(number_of_threads)
    results = out_pool.map(build_xls, processes)
    out_pool.close()
    out_pool.join()  # Wait until all threads are finished


def build_xls((outfile, signals, path_dir, file_name, file_number)):
    # The following may be different depending on PSSE version
    os.environ['PATH'] = PSSE33BINPATH + ';' + os.environ['PATH']
    sys.path.append('C:/Program Files (x86)/PTI/PSSEUniversity33/pssbin')
    sys.path.append('C:/Program Files (x86)/PTI/PSSEUniversity33/psslib')

    # sys.path.insert(0, PSSE33BINPATH)

    import dyntools
    # print(outfile)
    excel_file = path_dir + "\\sheets\\" + file_name + str(file_number) + '.xlsx'
    achnf = dyntools.CHNF(outfile);
    achnf.xlsout(channels=signals, show=False, xlsfile=excel_file, outfile='', sheet ='', overwritesheet=True);
    # dyntools.CHNF(outfile).xlsout(channels=signals, show=False, xlsfile=excel_file, outfile='', sheet='', overwritesheet=True)
    time.sleep(20)
#time.sleep(20) #this was used for testing

if __name__=='__main__':
    main()