#from multiprocessing.pool import ThreadPool
#from multiprocessing.dummy import Pool as ThreadPool
from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool
import tweepy
import random
import json
import simplejson
import re
from tweepy import OAuthHandler
from tweepy.auth import OAuthHandler
import gc
import sys
import time
from datetime import datetime
import threading
import psycopg2
# Consumer keys and access tokens, used for OAuth
consumer_key = "xxxxxxxx"
consumer_secret = "xxxxxxxx"
access_token = "xxxxxxxxxx"
access_token_secret = "xxxxxxx"
# OAuth process, using the keys and tokens
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
# Creation of the actual interface, using authentication
try:
api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True, retry_count=10, retry_delay=60)
#auth = api.VerifyCredentials()
except Exception as e:
print("Exception is occured",e)
count =0
def twitterjson(string):
results = tweepy.Cursor(api.user_timeline, screen_name=sm_handle, count=100).items()
for tweet in results:
with open(sm_handle+'.json', 'w') as outfile:
outfile.write(json.dumps(tweet._json))
print (tweet)
return 1;
if __name__ == "__main__":
while True:
try:
# Establishing a PSQL connection
conn = psycopg2.connect("dbname='sample' user='postgres' host='localhost' port='5432'")
print("opened database successfully")
# create a psycopg2 cursor that can execute queries
cursor = conn.cursor()
# twitter_users_query = "select twitter_user_id, twitter_screen_name from twitter_users;"
twitter_users_query = "select usernames from test;"
cursor.execute(twitter_users_query)
#pool = ThreadPool(10)
rows = cursor.fetchall()
names=[]
for row in rows:
sm_handle = row[0]
names.append(sm_handle)
print("sm_handles: "+sm_handle)
threads = []
# for i in rows:
for name in names:
print ("name:"+name)
except Exception as e:
print("unable to connect to the database : ", e)
try:
pool = ThreadPool(100)
gc.collect()
#pool.terminate()
results=pool.map(twitterjson, names)
pool.terminate(100)
print ("Completed"+results)
pool.terminate()
pool.close()
pool.join()
print("This is sleeping for 1 minute..............")
#time.sleep(60)
print('Done')
#pool.terminate()
#pool.close()
#pool.join()
#pool.terminate()
except Exception as e:
print("Exception is occured",e)
pool.terminate();
How to reallocating memory space in multithreading program in python ?
Users browsing this thread: 1 Guest(s)
