KnowHow

import queue
import threading
import logging
import time

logging.basicConfig(
    level=logging.DEBUG, format='%(threadName)s: %(message)s')


def create_log_file(filename, num):
    with open(filename, 'w') as f:
        for i in range(num):
            f.write(f'{i} Hello\n')


def read_log_to_queue(filename) -> queue:
    logfile = queue.Queue()
    with open(filename, 'r') as f:
        rows = f.read().split('\n')

    for row in rows:
        # print(row)
        logfile.put(row)

    return logfile


def worker(_queue):
    logging.debug('start')
    while True:
        item = _queue.get()
        if item is None:
            break
        logging.debug(item)
        _queue.task_done()
    logging.debug('end')


if __name__ == '__main__':
    logging.debug('start')
    NUM_OF_THREAD = 10
    file_name = 'sample.log'
    # create_log_file(file_name, 100000)

    start = time.time()
    queue = read_log_to_queue(file_name)

    ts = []
    for _ in range(NUM_OF_THREAD):
        t = threading.Thread(target=worker, args=(queue,))
        t.start()
        ts.append(t)

    logging.debug('tasks are not done')
    queue.join()
    logging.debug('tasks are done')

    [queue.put(None) for _ in range(len(ts))]
    [t.join() for t in ts]

    end = time.time()
    logging.debug('end')
    print('thread time: {: 4f}\n'.format(end - start))