Process (pool,map) strange behaviour

[maverick76]

Still editing sorry

import configparser,os,sys,time,mysql.connector,logging,io,locale,threading,queue
from gen_rapport import *
from multiprocessing import Pool,Lock, Process, Queue, current_process
from threading import Thread
import plotly.graph_objects as go

#Recuperation du fichier conf.ini
Config = configparser.ConfigParser()
Config.read(sys.argv[1])
num_process=sys.argv[2]
# print('proc='+num_process)
robotfolder=Config.get('dossier','robotfolder')

#Definition des logs logging.info('') et redirection de laffichage et des erreurs dans le fichier de log
log_path=Config.get('dossier','log_path')
log_file=Config.get('dossier','log_file')
logging.basicConfig(filename=log_path+log_file, filemode='w',level=logging.INFO,format='%(levelname)s:%(message)s')
f = open('data.csv','w')
f.write('nomrapport,time\n')
f.flush()
# sys.stdout = f
# sys.stderr = f


#Configuration de lutilisateur
NNI=Config.get('credentials','Login_Gardian')
Gardian=Config.get('credentials','MdP_Gardian')


#Configuration de lacces a la base de donnees
mydb = mysql.connector.connect(
    host=Config.get('credentials','SQL_IPaddress'),
    port=Config.get('credentials','SQL_Port'),
    user=Config.get('credentials','SQL_User'),
    password=Config.get('credentials','SQL_Password'))

base=Config.get('credentials','SQL_Base')
table=Config.get('credentials','SQL_Table')
max_threads=Config.get('thread','maxthreads')

#Recuperation du nombre darguments pour savoir quelles fonctions on lance
NmbArg = len(sys.argv)
# print(NmbArg)
#Affichage et formatage de la date et heure
locale.setlocale(locale.LC_TIME,'')
today = time.strftime("%d").lstrip("0").replace("0", "")

print('Commencement :'+str(time.asctime()))
tomonth = time.strftime("%B")
tomonth1 = tomonth[:1]
tomonth2 = tomonth[-1:]
toyear = time.strftime("%Y")
today = tomonth1+tomonth2+" "+toyear

def work_log(work_data):
	
	t = robot(NNI,Gardian,work_data[4],work_data[2],work_data[0],work_data[1],work_data[5],work_data[6],today)
	print(" Process %s Finished." % work_data[0])
	f.write(str(t[0])+','+str(t[1])+'\n')
	f.flush()	


def pool_handler():
	p = Pool(int(num_process))
	p.map(work_log, work)
	f.close()

if NmbArg == 4:

	if __name__ == '__main__':


		start = time.time()
		#Requete SQL pour recuperer les rapports a jouer
		mycursor = mydb.cursor()
		mycursor.execute("SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM "+base+"."+table+" where Type LIKE '%"+sys.argv[3]+"%' AND Executer='OUI'")
		#mycursor.execute("SELECT NomRapport,xlsfile,ImportFolder2,Type,ID, MinSize, Invites FROM "+base+"."+table+" where Type LIKE '%BI%' AND Executer='OUI'")
		print(("requete SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM "+base+"."+table+" where Type='"+sys.argv[3]+"' AND Executer='OUI'"))
		myresult = mycursor.fetchall()

		work = ()

		for rapports in myresult:
		
			
			NomRapport=rapports[0]
			xlsfile=rapports[1]
			ImportFolder=rapports[2]
			ID=rapports[4]
			MinSize=rapports[5]
			Invites=rapports[6]
			
			
			work = work + (rapports,)


		pool_handler()
		print("Fin:"+str(time.asctime()))

else:
	print('Usage : createbi.exe conf.ini PCA; ou bien createbi.exe conf.ini ID,ImportFolder,xlsfile,(Invites)')

Output:
PS C:\Users\d74074\Documents\Devs\Python\BIV6> python .\createbi.py .\conf.ini 6 BI
Commencement :Mon Feb  3 13:39:40 2020
requete SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM tabatta.l_app_createbi where Type='BI' AND Executer='OUI'
Commencement :Mon Feb  3 13:39:45 2020
Fichier present:BI6_CauseFigeageTardif
C:\Users\d74074\Documents\Devs\Python\BIV6\Imports\figeagetardif\PAL-tabatta_tot_cause_figeage_tardif.csv
Commencement :Mon Feb  3 13:39:45 2020
Fichier present:BI6_GenCloture
C:\Users\d74074\Documents\Devs\Python\BIV6\Imports\BITOTTypeCloture\PAL-Tabatta_BI_-_TOT_-_TypeCloture.csv
Commencement :Mon Feb  3 13:39:45 2020
doesnt exist:BI6_GenCompl2340
Commencement :Mon Feb  3 13:39:45 2020
doesnt exist:BI6_Gencomplt340
Commencement :Mon Feb  3 13:39:45 2020
doesnt exist:BI6_GenDatesTOT
Commencement :Mon Feb  3 13:39:45 2020

[maverick76]

Hello,

I have learnt Python from myself because at work, i had to automate some action on a web app(i am using selenium).
For that, i have a list of url in a database.

As i am a beginner, there is something i don't understand.

Last week, i have been able to start some process (the number is a command line argument).
These processes are a function, called 'robot()'.

It works!

But i am wondering, why the "print('Commencement :'+str(time.asctime()))" line 48 appear more than 1 time in the console?

because it is written only one time(line 48) in the script, and it is not in the 'work_log' function.

If someone can explain, i would be very happy !

next step for me, process which write on the same file....

import configparser,os,sys,time,mysql.connector,logging,io,locale,threading,queue
from gen_rapport import *
from multiprocessing import Pool,Lock, Process, Queue, current_process
from threading import Thread
import plotly.graph_objects as go

#Recuperation du fichier conf.ini
Config = configparser.ConfigParser()
Config.read(sys.argv[1])
num_process=sys.argv[2]
# print('proc='+num_process)
robotfolder=Config.get('dossier','robotfolder')

#Definition des logs logging.info('') et redirection de laffichage et des erreurs dans le fichier de log
log_path=Config.get('dossier','log_path')
log_file=Config.get('dossier','log_file')
logging.basicConfig(filename=log_path+log_file, filemode='w',level=logging.INFO,format='%(levelname)s:%(message)s')
f = open('data.csv','w')
f.write('nomrapport,time\n')
f.flush()
# sys.stdout = f
# sys.stderr = f


#Configuration de lutilisateur
Login=Config.get('credentials','Login')
MdP=Config.get('credentials','MdP')


#Configuration de lacces a la base de donnees
mydb = mysql.connector.connect(
    host=Config.get('credentials','SQL_IPaddress'),
    port=Config.get('credentials','SQL_Port'),
    user=Config.get('credentials','SQL_User'),
    password=Config.get('credentials','SQL_Password'))

base=Config.get('credentials','SQL_Base')
table=Config.get('credentials','SQL_Table')
max_threads=Config.get('thread','maxthreads')

#Recuperation du nombre darguments pour savoir quelles fonctions on lance
NmbArg = len(sys.argv)
# print(NmbArg)
#Affichage et formatage de la date et heure
locale.setlocale(locale.LC_TIME,'')
today = time.strftime("%d").lstrip("0").replace("0", "")

print('Commencement :'+str(time.asctime()))
tomonth = time.strftime("%B")
tomonth1 = tomonth[:1]
tomonth2 = tomonth[-1:]
toyear = time.strftime("%Y")
today = tomonth1+tomonth2+" "+toyear

def work_log(work_data):
	
	t = robot(Login,MdP,work_data[4],work_data[2],work_data[0],work_data[1],work_data[5],work_data[6],today)
	print(" Process %s Finished." % work_data[0])
	f.write(str(t[0])+','+str(t[1])+'\n')
	f.flush()	


def pool_handler():
	p = Pool(int(num_process))
	p.map(work_log, work)
	f.close()

if NmbArg == 4:

	if __name__ == '__main__':


		start = time.time()
		#Requete SQL pour recuperer les rapports a jouer
		mycursor = mydb.cursor()
		mycursor.execute("SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM "+base+"."+table+" where Type LIKE '%"+sys.argv[3]+"%' AND Executer='OUI'")
		#mycursor.execute("SELECT NomRapport,xlsfile,ImportFolder2,Type,ID, MinSize, Invites FROM "+base+"."+table+" where Type LIKE '%BI%' AND Executer='OUI'")
		print(("requete SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM "+base+"."+table+" where Type='"+sys.argv[3]+"' AND Executer='OUI'"))
		myresult = mycursor.fetchall()

		work = ()

		for rapports in myresult:
		
			
			NomRapport=rapports[0]
			xlsfile=rapports[1]
			ImportFolder=rapports[2]
			ID=rapports[4]
			MinSize=rapports[5]
			Invites=rapports[6]
			
			
			work = work + (rapports,)


		pool_handler()
		print("Fin:"+str(time.asctime()))

else:
	print('Usage : createbi.exe conf.ini PCA; ou bien createbi.exe conf.ini ID,ImportFolder,xlsfile,(Invites)')

Output:
PS C:\Users\sssss\Documents\Devs\Python\ssss> python .\sssss.py .\conf.ini 6 BI
Commencement :Mon Feb  3 13:39:40 2020
requete SELECT NomRapport,xlsfile,ImportFolder2,Type,ID,MinSize, Invites FROM xxxxxxxxx where Type='BI' AND Executer='OUI'
Commencement :Mon Feb  3 13:39:45 2020
Fichier present:BI6_CauseFigeageTardif
C:\Users\sssss\Documents\Devs\Python\BIV6\Imports\figeagetardif\xxxxxx.csv
Commencement :Mon Feb  3 13:39:45 2020
Fichier present:BI6_GenCloture
C:\Users\sssss\Documents\Devs\Python\BIV6\Imports\BITOTTypeCloture\xcxcxcxc.csv
Commencement :Mon Feb  3 13:39:45 2020
doesnt exist:BI6_GenCompl2340
Commencement :Mon Feb  3 13:39:45 2020
doesnt exist:BI6_Gencomplt340
Commencement :Mon Feb  3 13:39:45 2020
doesnt exist:BI6_GenDatesTOT
Commencement :Mon Feb  3 13:39:45 2020