randomi with unlimited threads
finishes the 10k/50/25 bench in 20s on 8 threads known bugs: > might generate a few to many files if the total number of txt files isnt equally dividable by core count >the 'working, this might take a while' message is displayed after all files are created
This commit is contained in:
64
src/data_generators/randomiUnLi.py
Normal file
64
src/data_generators/randomiUnLi.py
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
# Meet randomI, a random data generator for test data generation
|
||||||
|
from random import randint
|
||||||
|
import multiprocessing
|
||||||
|
import time
|
||||||
|
from datetime import date
|
||||||
|
from typing import Any, Union
|
||||||
|
|
||||||
|
|
||||||
|
def main(div, a, b, o):
|
||||||
|
for x in range(0, int(div)):
|
||||||
|
writeFile(generateFile(a, b), x, o)
|
||||||
|
|
||||||
|
|
||||||
|
def generateFile(rows, lenghtOfRows):
|
||||||
|
content = []
|
||||||
|
for y in range(0, rows):
|
||||||
|
content.append(generateRow(lenghtOfRows))
|
||||||
|
return content
|
||||||
|
|
||||||
|
def generateRow(lenghtOfRows):
|
||||||
|
row = ""
|
||||||
|
for z in range(0, lenghtOfRows):
|
||||||
|
row = row + str(randint(0, 9))
|
||||||
|
return row
|
||||||
|
|
||||||
|
def writeFile(content, x, o):
|
||||||
|
file = open("testdata/file" + str(x+o) + ".txt", "w")
|
||||||
|
for w in range(0, len(content)):
|
||||||
|
file.write(content[w] + "\n")
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
print("Hello World")
|
||||||
|
i = int(input("How many units would you like to generate? "))
|
||||||
|
a = int(input("How many rows should each unit have? "))
|
||||||
|
b = int(input("How long should each row be? "))
|
||||||
|
c = int(input("How many threads do you want to use?"))
|
||||||
|
print('preparing')
|
||||||
|
counter = int(0)
|
||||||
|
div = i/c
|
||||||
|
|
||||||
|
while counter < c:
|
||||||
|
o = div * counter
|
||||||
|
print('thread ' + str(counter) + ' set to start at ' + str(o))
|
||||||
|
globals()["p" + str(counter)] = multiprocessing.Process(target=main, args=(div, a, b, o))
|
||||||
|
counter = counter +1
|
||||||
|
counter = int(0)
|
||||||
|
start_time = time.time()
|
||||||
|
print('starting')
|
||||||
|
|
||||||
|
while counter < c:
|
||||||
|
globals()["p" + str(counter)].start()
|
||||||
|
print('thread number ' +str(counter) + ' just started')
|
||||||
|
counter = counter + 1
|
||||||
|
counter = int(0)
|
||||||
|
|
||||||
|
while counter < c:
|
||||||
|
globals()["p" + str(counter)].join()
|
||||||
|
print('thread number ' + str(counter) + ' just finished')
|
||||||
|
counter = counter + 1
|
||||||
|
print('working, this might take a while')
|
||||||
|
print("Data is generated. Have fun!")
|
||||||
|
sec = time.time() - start_time
|
||||||
|
minutes = sec / 60
|
||||||
|
print("randomI took " + str(sec) + " seconds (" + str(minutes) + " minutes) for execution.")
|
||||||
Reference in New Issue
Block a user